training fix. add global settings
This commit is contained in:
@@ -1 +1 @@
|
||||
# Services module
|
||||
# Services module
|
||||
|
||||
@@ -1,92 +1,92 @@
|
||||
const API_URL = 'http://192.168.1.19:8080/api';
|
||||
const API_TOKEN = 'c1cef980b7c73004f4ee880a42839313b863869f';
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
async function fetchLableStudioProject(projectid) {
|
||||
// 1. Trigger export
|
||||
const exportUrl = `${API_URL}/projects/${projectid}/export?exportType=JSON_MIN`;
|
||||
const headers = { Authorization: `Token ${API_TOKEN}` };
|
||||
let res = await fetch(exportUrl, { headers });
|
||||
if (!res.ok) {
|
||||
let errorText = await res.text().catch(() => '');
|
||||
console.error(`Failed to trigger export: ${res.status} ${res.statusText} - ${errorText}`);
|
||||
throw new Error(`Failed to trigger export: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
let data = await res.json();
|
||||
// If data is an array, it's ready
|
||||
if (Array.isArray(data)) return data;
|
||||
// If not, poll for the export file
|
||||
let fileUrl = data.download_url || data.url || null;
|
||||
let tries = 0;
|
||||
while (!fileUrl && tries < 20) {
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
res = await fetch(exportUrl, { headers });
|
||||
if (!res.ok) {
|
||||
let errorText = await res.text().catch(() => '');
|
||||
console.error(`Failed to poll export: ${res.status} ${res.statusText} - ${errorText}`);
|
||||
throw new Error(`Failed to poll export: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
data = await res.json();
|
||||
fileUrl = data.download_url || data.url || null;
|
||||
tries++;
|
||||
}
|
||||
if (!fileUrl) throw new Error('Label Studio export did not become ready');
|
||||
// 2. Download the export file
|
||||
res = await fetch(fileUrl.startsWith('http') ? fileUrl : `${API_URL.replace('/api','')}${fileUrl}`, { headers });
|
||||
if (!res.ok) {
|
||||
let errorText = await res.text().catch(() => '');
|
||||
console.error(`Failed to download export: ${res.status} ${res.statusText} - ${errorText}`);
|
||||
throw new Error(`Failed to download export: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
return await res.json();
|
||||
}
|
||||
|
||||
|
||||
|
||||
async function fetchProjectIdsAndTitles() {
|
||||
try {
|
||||
const response = await fetch(`${API_URL}/projects/`, {
|
||||
headers: {
|
||||
'Authorization': `Token ${API_TOKEN}`,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
let errorText = await response.text().catch(() => '');
|
||||
console.error(`Failed to fetch projects: ${response.status} ${response.statusText} - ${errorText}`);
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.results || !Array.isArray(data.results)) {
|
||||
throw new Error('API response does not contain results array');
|
||||
}
|
||||
|
||||
// Extract id and title from each project
|
||||
const projects = data.results.map(project => ({
|
||||
id: project.id,
|
||||
title: project.title
|
||||
}));
|
||||
console.log(projects)
|
||||
return projects;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch projects:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { fetchLableStudioProject, fetchProjectIdsAndTitles };
|
||||
|
||||
|
||||
|
||||
//getLableStudioProject(20)
|
||||
//fetchProjectIdsAndTitles()
|
||||
const API_URL = 'http://192.168.1.19:8080/api';
|
||||
const API_TOKEN = 'c1cef980b7c73004f4ee880a42839313b863869f';
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
async function fetchLableStudioProject(projectid) {
|
||||
// 1. Trigger export
|
||||
const exportUrl = `${API_URL}/projects/${projectid}/export?exportType=JSON_MIN`;
|
||||
const headers = { Authorization: `Token ${API_TOKEN}` };
|
||||
let res = await fetch(exportUrl, { headers });
|
||||
if (!res.ok) {
|
||||
let errorText = await res.text().catch(() => '');
|
||||
console.error(`Failed to trigger export: ${res.status} ${res.statusText} - ${errorText}`);
|
||||
throw new Error(`Failed to trigger export: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
let data = await res.json();
|
||||
// If data is an array, it's ready
|
||||
if (Array.isArray(data)) return data;
|
||||
// If not, poll for the export file
|
||||
let fileUrl = data.download_url || data.url || null;
|
||||
let tries = 0;
|
||||
while (!fileUrl && tries < 20) {
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
res = await fetch(exportUrl, { headers });
|
||||
if (!res.ok) {
|
||||
let errorText = await res.text().catch(() => '');
|
||||
console.error(`Failed to poll export: ${res.status} ${res.statusText} - ${errorText}`);
|
||||
throw new Error(`Failed to poll export: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
data = await res.json();
|
||||
fileUrl = data.download_url || data.url || null;
|
||||
tries++;
|
||||
}
|
||||
if (!fileUrl) throw new Error('Label Studio export did not become ready');
|
||||
// 2. Download the export file
|
||||
res = await fetch(fileUrl.startsWith('http') ? fileUrl : `${API_URL.replace('/api','')}${fileUrl}`, { headers });
|
||||
if (!res.ok) {
|
||||
let errorText = await res.text().catch(() => '');
|
||||
console.error(`Failed to download export: ${res.status} ${res.statusText} - ${errorText}`);
|
||||
throw new Error(`Failed to download export: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
return await res.json();
|
||||
}
|
||||
|
||||
|
||||
|
||||
async function fetchProjectIdsAndTitles() {
|
||||
try {
|
||||
const response = await fetch(`${API_URL}/projects/`, {
|
||||
headers: {
|
||||
'Authorization': `Token ${API_TOKEN}`,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
let errorText = await response.text().catch(() => '');
|
||||
console.error(`Failed to fetch projects: ${response.status} ${response.statusText} - ${errorText}`);
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.results || !Array.isArray(data.results)) {
|
||||
throw new Error('API response does not contain results array');
|
||||
}
|
||||
|
||||
// Extract id and title from each project
|
||||
const projects = data.results.map(project => ({
|
||||
id: project.id,
|
||||
title: project.title
|
||||
}));
|
||||
console.log(projects)
|
||||
return projects;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch projects:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { fetchLableStudioProject, fetchProjectIdsAndTitles };
|
||||
|
||||
|
||||
|
||||
//getLableStudioProject(20)
|
||||
//fetchProjectIdsAndTitles()
|
||||
|
||||
@@ -1,85 +1,93 @@
|
||||
import requests
|
||||
import time
|
||||
|
||||
API_URL = 'http://192.168.1.19:8080/api'
|
||||
API_TOKEN = 'c1cef980b7c73004f4ee880a42839313b863869f'
|
||||
|
||||
def fetch_label_studio_project(project_id):
|
||||
"""Fetch Label Studio project annotations"""
|
||||
export_url = f'{API_URL}/projects/{project_id}/export?exportType=JSON_MIN'
|
||||
headers = {'Authorization': f'Token {API_TOKEN}'}
|
||||
|
||||
# Trigger export
|
||||
res = requests.get(export_url, headers=headers)
|
||||
if not res.ok:
|
||||
error_text = res.text if res.text else ''
|
||||
print(f'Failed to trigger export: {res.status_code} {res.reason} - {error_text}')
|
||||
raise Exception(f'Failed to trigger export: {res.status_code} {res.reason}')
|
||||
|
||||
data = res.json()
|
||||
|
||||
# If data is an array, it's ready
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
|
||||
# If not, poll for the export file
|
||||
file_url = data.get('download_url') or data.get('url')
|
||||
tries = 0
|
||||
|
||||
while not file_url and tries < 20:
|
||||
time.sleep(2)
|
||||
res = requests.get(export_url, headers=headers)
|
||||
if not res.ok:
|
||||
error_text = res.text if res.text else ''
|
||||
print(f'Failed to poll export: {res.status_code} {res.reason} - {error_text}')
|
||||
raise Exception(f'Failed to poll export: {res.status_code} {res.reason}')
|
||||
|
||||
data = res.json()
|
||||
file_url = data.get('download_url') or data.get('url')
|
||||
tries += 1
|
||||
|
||||
if not file_url:
|
||||
raise Exception('Label Studio export did not become ready')
|
||||
|
||||
# Download the export file
|
||||
full_url = file_url if file_url.startswith('http') else f"{API_URL.replace('/api', '')}{file_url}"
|
||||
res = requests.get(full_url, headers=headers)
|
||||
if not res.ok:
|
||||
error_text = res.text if res.text else ''
|
||||
print(f'Failed to download export: {res.status_code} {res.reason} - {error_text}')
|
||||
raise Exception(f'Failed to download export: {res.status_code} {res.reason}')
|
||||
|
||||
return res.json()
|
||||
|
||||
def fetch_project_ids_and_titles():
|
||||
"""Fetch all Label Studio project IDs and titles"""
|
||||
try:
|
||||
response = requests.get(
|
||||
f'{API_URL}/projects/',
|
||||
headers={
|
||||
'Authorization': f'Token {API_TOKEN}',
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
)
|
||||
|
||||
if not response.ok:
|
||||
error_text = response.text if response.text else ''
|
||||
print(f'Failed to fetch projects: {response.status_code} {response.reason} - {error_text}')
|
||||
raise Exception(f'HTTP error! status: {response.status_code}')
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'results' not in data or not isinstance(data['results'], list):
|
||||
raise Exception('API response does not contain results array')
|
||||
|
||||
# Extract id and title from each project
|
||||
projects = [
|
||||
{'id': project['id'], 'title': project['title']}
|
||||
for project in data['results']
|
||||
]
|
||||
print(projects)
|
||||
return projects
|
||||
|
||||
except Exception as error:
|
||||
print(f'Failed to fetch projects: {error}')
|
||||
return []
|
||||
import requests
|
||||
import time
|
||||
from services.settings_service import get_setting
|
||||
|
||||
def get_api_credentials():
|
||||
"""Get Label Studio API credentials from settings"""
|
||||
api_url = get_setting('labelstudio_api_url', 'http://192.168.1.19:8080/api')
|
||||
api_token = get_setting('labelstudio_api_token', 'c1cef980b7c73004f4ee880a42839313b863869f')
|
||||
return api_url, api_token
|
||||
|
||||
def fetch_label_studio_project(project_id):
|
||||
"""Fetch Label Studio project annotations"""
|
||||
API_URL, API_TOKEN = get_api_credentials()
|
||||
|
||||
export_url = f'{API_URL}/projects/{project_id}/export?exportType=JSON_MIN'
|
||||
headers = {'Authorization': f'Token {API_TOKEN}'}
|
||||
|
||||
# Trigger export
|
||||
res = requests.get(export_url, headers=headers)
|
||||
if not res.ok:
|
||||
error_text = res.text if res.text else ''
|
||||
print(f'Failed to trigger export: {res.status_code} {res.reason} - {error_text}')
|
||||
raise Exception(f'Failed to trigger export: {res.status_code} {res.reason}')
|
||||
|
||||
data = res.json()
|
||||
|
||||
# If data is an array, it's ready
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
|
||||
# If not, poll for the export file
|
||||
file_url = data.get('download_url') or data.get('url')
|
||||
tries = 0
|
||||
|
||||
while not file_url and tries < 20:
|
||||
time.sleep(2)
|
||||
res = requests.get(export_url, headers=headers)
|
||||
if not res.ok:
|
||||
error_text = res.text if res.text else ''
|
||||
print(f'Failed to poll export: {res.status_code} {res.reason} - {error_text}')
|
||||
raise Exception(f'Failed to poll export: {res.status_code} {res.reason}')
|
||||
|
||||
data = res.json()
|
||||
file_url = data.get('download_url') or data.get('url')
|
||||
tries += 1
|
||||
|
||||
if not file_url:
|
||||
raise Exception('Label Studio export did not become ready')
|
||||
|
||||
# Download the export file
|
||||
full_url = file_url if file_url.startswith('http') else f"{API_URL.replace('/api', '')}{file_url}"
|
||||
res = requests.get(full_url, headers=headers)
|
||||
if not res.ok:
|
||||
error_text = res.text if res.text else ''
|
||||
print(f'Failed to download export: {res.status_code} {res.reason} - {error_text}')
|
||||
raise Exception(f'Failed to download export: {res.status_code} {res.reason}')
|
||||
|
||||
return res.json()
|
||||
|
||||
def fetch_project_ids_and_titles():
|
||||
"""Fetch all Label Studio project IDs and titles"""
|
||||
API_URL, API_TOKEN = get_api_credentials()
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
f'{API_URL}/projects/',
|
||||
headers={
|
||||
'Authorization': f'Token {API_TOKEN}',
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
)
|
||||
|
||||
if not response.ok:
|
||||
error_text = response.text if response.text else ''
|
||||
print(f'Failed to fetch projects: {response.status_code} {response.reason} - {error_text}')
|
||||
raise Exception(f'HTTP error! status: {response.status_code}')
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'results' not in data or not isinstance(data['results'], list):
|
||||
raise Exception('API response does not contain results array')
|
||||
|
||||
# Extract id and title from each project
|
||||
projects = [
|
||||
{'id': project['id'], 'title': project['title']}
|
||||
for project in data['results']
|
||||
]
|
||||
print(projects)
|
||||
return projects
|
||||
|
||||
except Exception as error:
|
||||
print(f'Failed to fetch projects: {error}')
|
||||
return []
|
||||
|
||||
@@ -1,176 +1,176 @@
|
||||
const TrainingProject = require('../models/TrainingProject.js');
|
||||
const TrainingProjectDetails = require('../models/TrainingProjectDetails.js')
|
||||
const LabelStudioProject = require('../models/LabelStudioProject.js')
|
||||
const Annotation = require('../models/Annotation.js')
|
||||
const Images = require('../models/Images.js')
|
||||
const fs = require('fs');
|
||||
|
||||
|
||||
async function generateTrainingJson(trainingId){
|
||||
// trainingId is now project_details_id
|
||||
const trainingProjectDetails = await TrainingProjectDetails.findByPk(trainingId);
|
||||
if (!trainingProjectDetails) throw new Error('No TrainingProjectDetails found for project_details_id ' + trainingId);
|
||||
const detailsObj = trainingProjectDetails.get({ plain: true });
|
||||
// Get parent project for name
|
||||
const trainingProject = await TrainingProject.findByPk(detailsObj.project_id);
|
||||
// Get split percentages (assume they are stored as train_percent, valid_percent, test_percent)
|
||||
const trainPercent = detailsObj.train_percent || 85;
|
||||
const validPercent = detailsObj.valid_percent || 10;
|
||||
const testPercent = detailsObj.test_percent || 5;
|
||||
|
||||
let cocoImages = [];
|
||||
let cocoAnnotations = [];
|
||||
let cocoCategories = [];
|
||||
let categoryMap = {};
|
||||
let categoryId = 0;
|
||||
let imageid = 0;
|
||||
let annotationid = 0;
|
||||
|
||||
for (const cls of detailsObj.class_map) {
|
||||
const asgMap = [];
|
||||
const listAsg = cls[1];
|
||||
for(const asg of listAsg){
|
||||
asgMap.push ({ original: asg[0], mapped: asg[1] });
|
||||
// Build category list and mapping
|
||||
if (asg[1] && !(asg[1] in categoryMap)) {
|
||||
categoryMap[asg[1]] = categoryId;
|
||||
cocoCategories.push({ id: categoryId, name: asg[1], supercategory: '' });
|
||||
categoryId++;
|
||||
}
|
||||
}
|
||||
const images = await Images.findAll({ where: { project_id: cls[0] } });
|
||||
for(const image of images){
|
||||
imageid += 1;
|
||||
let fileName = image.image_path;
|
||||
if (fileName.includes('%20')) {
|
||||
fileName = fileName.replace(/%20/g, ' ');
|
||||
}
|
||||
if (fileName && fileName.startsWith('/data/local-files/?d=')) {
|
||||
fileName = fileName.replace('/data/local-files/?d=', '');
|
||||
fileName = fileName.replace('/home/kitraining/home/kitraining/', '');
|
||||
}
|
||||
if (fileName && fileName.startsWith('home/kitraining/To_Annotate/')) {
|
||||
fileName = fileName.replace('home/kitraining/To_Annotate/','');
|
||||
}
|
||||
// Get annotations for this image
|
||||
const annotations = await Annotation.findAll({ where: { image_id: image.image_id } });
|
||||
// Use image.width and image.height from DB (populated from original_width/original_height)
|
||||
cocoImages.push({
|
||||
id: imageid,
|
||||
file_name: fileName,
|
||||
width: image.width || 0,
|
||||
height: image.height || 0
|
||||
});
|
||||
for (const annotation of annotations) {
|
||||
// Translate class name using asgMap
|
||||
let mappedClass = annotation.Label;
|
||||
for (const mapEntry of asgMap) {
|
||||
if (annotation.Label === mapEntry.original) {
|
||||
mappedClass = mapEntry.mapped;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Only add annotation if mappedClass is valid
|
||||
if (mappedClass && mappedClass in categoryMap) {
|
||||
annotationid += 1;
|
||||
let area = 0;
|
||||
if (annotation.width && annotation.height) {
|
||||
area = annotation.width * annotation.height;
|
||||
}
|
||||
cocoAnnotations.push({
|
||||
id: annotationid,
|
||||
image_id: imageid,
|
||||
category_id: categoryMap[mappedClass],
|
||||
bbox: [annotation.x, annotation.y, annotation.width, annotation.height],
|
||||
area: area,
|
||||
iscrowd: annotation.iscrowd || 0
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shuffle images for random split using seed
|
||||
function seededRandom(seed) {
|
||||
let x = Math.sin(seed++) * 10000;
|
||||
return x - Math.floor(x);
|
||||
}
|
||||
function shuffle(array, seed) {
|
||||
for (let i = array.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(seededRandom(seed + i) * (i + 1));
|
||||
[array[i], array[j]] = [array[j], array[i]];
|
||||
}
|
||||
}
|
||||
// Use seed from detailsObj if present, else default to 42
|
||||
const splitSeed = detailsObj.seed !== undefined && detailsObj.seed !== null ? Number(detailsObj.seed) : 42;
|
||||
shuffle(cocoImages, splitSeed);
|
||||
|
||||
// Split images
|
||||
const totalImages = cocoImages.length;
|
||||
const trainCount = Math.floor(totalImages * trainPercent / 100);
|
||||
const validCount = Math.floor(totalImages * validPercent / 100);
|
||||
const testCount = totalImages - trainCount - validCount;
|
||||
|
||||
const trainImages = cocoImages.slice(0, trainCount);
|
||||
const validImages = cocoImages.slice(trainCount, trainCount + validCount);
|
||||
const testImages = cocoImages.slice(trainCount + validCount);
|
||||
|
||||
// Helper to get image ids for each split
|
||||
const trainImageIds = new Set(trainImages.map(img => img.id));
|
||||
const validImageIds = new Set(validImages.map(img => img.id));
|
||||
const testImageIds = new Set(testImages.map(img => img.id));
|
||||
|
||||
// Split annotations
|
||||
const trainAnnotations = cocoAnnotations.filter(ann => trainImageIds.has(ann.image_id));
|
||||
const validAnnotations = cocoAnnotations.filter(ann => validImageIds.has(ann.image_id));
|
||||
const testAnnotations = cocoAnnotations.filter(ann => testImageIds.has(ann.image_id));
|
||||
|
||||
// Build final COCO JSONs with info section
|
||||
const buildCocoJson = (images, annotations, categories) => ({
|
||||
images,
|
||||
annotations,
|
||||
categories
|
||||
});
|
||||
|
||||
// Build COCO JSONs with info section
|
||||
const trainJson = buildCocoJson(trainImages, trainAnnotations, cocoCategories);
|
||||
const validJson = buildCocoJson(validImages, validAnnotations, cocoCategories);
|
||||
const testJson = buildCocoJson(testImages, testAnnotations, cocoCategories);
|
||||
|
||||
// Create output directory: projectname/trainingid/annotations
|
||||
const projectName = trainingProject && trainingProject.name ? trainingProject.name.replace(/\s+/g, '_') : `project_${detailsObj.project_id}`;
|
||||
const outDir = `${projectName}/${trainingId}`;
|
||||
const annotationsDir = `/home/kitraining/To_Annotate/annotations`;
|
||||
if (!fs.existsSync(annotationsDir)) {
|
||||
fs.mkdirSync(annotationsDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Write to files in the annotations directory
|
||||
const trainPath = `${annotationsDir}/coco_project_${trainingId}_train.json`;
|
||||
const validPath = `${annotationsDir}/coco_project_${trainingId}_valid.json`;
|
||||
const testPath = `${annotationsDir}/coco_project_${trainingId}_test.json`;
|
||||
fs.writeFileSync(trainPath, JSON.stringify(trainJson, null, 2));
|
||||
fs.writeFileSync(validPath, JSON.stringify(validJson, null, 2));
|
||||
fs.writeFileSync(testPath, JSON.stringify(testJson, null, 2));
|
||||
console.log(`COCO JSON splits written to ${annotationsDir} for trainingId ${trainingId}`);
|
||||
|
||||
|
||||
|
||||
// Also generate inference exp.py in the same output directory as exp.py (project folder in workspace)
|
||||
const { generateYoloxInferenceExp } = require('./generate-yolox-exp');
|
||||
const path = require('path');
|
||||
const projectFolder = path.join(__dirname, '..', projectName, String(trainingId));
|
||||
if (!fs.existsSync(projectFolder)) {
|
||||
fs.mkdirSync(projectFolder, { recursive: true });
|
||||
}
|
||||
const inferenceExpPath = path.join(projectFolder, 'exp_infer.py');
|
||||
generateYoloxInferenceExp(trainingId).then(expContent => {
|
||||
fs.writeFileSync(inferenceExpPath, expContent);
|
||||
console.log(`Inference exp.py written to ${inferenceExpPath}`);
|
||||
}).catch(err => {
|
||||
console.error('Failed to generate inference exp.py:', err);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
const TrainingProject = require('../models/TrainingProject.js');
|
||||
const TrainingProjectDetails = require('../models/TrainingProjectDetails.js')
|
||||
const LabelStudioProject = require('../models/LabelStudioProject.js')
|
||||
const Annotation = require('../models/Annotation.js')
|
||||
const Images = require('../models/Images.js')
|
||||
const fs = require('fs');
|
||||
|
||||
|
||||
async function generateTrainingJson(trainingId){
|
||||
// trainingId is now project_details_id
|
||||
const trainingProjectDetails = await TrainingProjectDetails.findByPk(trainingId);
|
||||
if (!trainingProjectDetails) throw new Error('No TrainingProjectDetails found for project_details_id ' + trainingId);
|
||||
const detailsObj = trainingProjectDetails.get({ plain: true });
|
||||
// Get parent project for name
|
||||
const trainingProject = await TrainingProject.findByPk(detailsObj.project_id);
|
||||
// Get split percentages (assume they are stored as train_percent, valid_percent, test_percent)
|
||||
const trainPercent = detailsObj.train_percent || 85;
|
||||
const validPercent = detailsObj.valid_percent || 10;
|
||||
const testPercent = detailsObj.test_percent || 5;
|
||||
|
||||
let cocoImages = [];
|
||||
let cocoAnnotations = [];
|
||||
let cocoCategories = [];
|
||||
let categoryMap = {};
|
||||
let categoryId = 0;
|
||||
let imageid = 0;
|
||||
let annotationid = 0;
|
||||
|
||||
for (const cls of detailsObj.class_map) {
|
||||
const asgMap = [];
|
||||
const listAsg = cls[1];
|
||||
for(const asg of listAsg){
|
||||
asgMap.push ({ original: asg[0], mapped: asg[1] });
|
||||
// Build category list and mapping
|
||||
if (asg[1] && !(asg[1] in categoryMap)) {
|
||||
categoryMap[asg[1]] = categoryId;
|
||||
cocoCategories.push({ id: categoryId, name: asg[1], supercategory: '' });
|
||||
categoryId++;
|
||||
}
|
||||
}
|
||||
const images = await Images.findAll({ where: { project_id: cls[0] } });
|
||||
for(const image of images){
|
||||
imageid += 1;
|
||||
let fileName = image.image_path;
|
||||
if (fileName.includes('%20')) {
|
||||
fileName = fileName.replace(/%20/g, ' ');
|
||||
}
|
||||
if (fileName && fileName.startsWith('/data/local-files/?d=')) {
|
||||
fileName = fileName.replace('/data/local-files/?d=', '');
|
||||
fileName = fileName.replace('/home/kitraining/home/kitraining/', '');
|
||||
}
|
||||
if (fileName && fileName.startsWith('home/kitraining/To_Annotate/')) {
|
||||
fileName = fileName.replace('home/kitraining/To_Annotate/','');
|
||||
}
|
||||
// Get annotations for this image
|
||||
const annotations = await Annotation.findAll({ where: { image_id: image.image_id } });
|
||||
// Use image.width and image.height from DB (populated from original_width/original_height)
|
||||
cocoImages.push({
|
||||
id: imageid,
|
||||
file_name: fileName,
|
||||
width: image.width || 0,
|
||||
height: image.height || 0
|
||||
});
|
||||
for (const annotation of annotations) {
|
||||
// Translate class name using asgMap
|
||||
let mappedClass = annotation.Label;
|
||||
for (const mapEntry of asgMap) {
|
||||
if (annotation.Label === mapEntry.original) {
|
||||
mappedClass = mapEntry.mapped;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Only add annotation if mappedClass is valid
|
||||
if (mappedClass && mappedClass in categoryMap) {
|
||||
annotationid += 1;
|
||||
let area = 0;
|
||||
if (annotation.width && annotation.height) {
|
||||
area = annotation.width * annotation.height;
|
||||
}
|
||||
cocoAnnotations.push({
|
||||
id: annotationid,
|
||||
image_id: imageid,
|
||||
category_id: categoryMap[mappedClass],
|
||||
bbox: [annotation.x, annotation.y, annotation.width, annotation.height],
|
||||
area: area,
|
||||
iscrowd: annotation.iscrowd || 0
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shuffle images for random split using seed
|
||||
function seededRandom(seed) {
|
||||
let x = Math.sin(seed++) * 10000;
|
||||
return x - Math.floor(x);
|
||||
}
|
||||
function shuffle(array, seed) {
|
||||
for (let i = array.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(seededRandom(seed + i) * (i + 1));
|
||||
[array[i], array[j]] = [array[j], array[i]];
|
||||
}
|
||||
}
|
||||
// Use seed from detailsObj if present, else default to 42
|
||||
const splitSeed = detailsObj.seed !== undefined && detailsObj.seed !== null ? Number(detailsObj.seed) : 42;
|
||||
shuffle(cocoImages, splitSeed);
|
||||
|
||||
// Split images
|
||||
const totalImages = cocoImages.length;
|
||||
const trainCount = Math.floor(totalImages * trainPercent / 100);
|
||||
const validCount = Math.floor(totalImages * validPercent / 100);
|
||||
const testCount = totalImages - trainCount - validCount;
|
||||
|
||||
const trainImages = cocoImages.slice(0, trainCount);
|
||||
const validImages = cocoImages.slice(trainCount, trainCount + validCount);
|
||||
const testImages = cocoImages.slice(trainCount + validCount);
|
||||
|
||||
// Helper to get image ids for each split
|
||||
const trainImageIds = new Set(trainImages.map(img => img.id));
|
||||
const validImageIds = new Set(validImages.map(img => img.id));
|
||||
const testImageIds = new Set(testImages.map(img => img.id));
|
||||
|
||||
// Split annotations
|
||||
const trainAnnotations = cocoAnnotations.filter(ann => trainImageIds.has(ann.image_id));
|
||||
const validAnnotations = cocoAnnotations.filter(ann => validImageIds.has(ann.image_id));
|
||||
const testAnnotations = cocoAnnotations.filter(ann => testImageIds.has(ann.image_id));
|
||||
|
||||
// Build final COCO JSONs with info section
|
||||
const buildCocoJson = (images, annotations, categories) => ({
|
||||
images,
|
||||
annotations,
|
||||
categories
|
||||
});
|
||||
|
||||
// Build COCO JSONs with info section
|
||||
const trainJson = buildCocoJson(trainImages, trainAnnotations, cocoCategories);
|
||||
const validJson = buildCocoJson(validImages, validAnnotations, cocoCategories);
|
||||
const testJson = buildCocoJson(testImages, testAnnotations, cocoCategories);
|
||||
|
||||
// Create output directory: projectname/trainingid/annotations
|
||||
const projectName = trainingProject && trainingProject.name ? trainingProject.name.replace(/\s+/g, '_') : `project_${detailsObj.project_id}`;
|
||||
const outDir = `${projectName}/${trainingId}`;
|
||||
const annotationsDir = `/home/kitraining/To_Annotate/annotations`;
|
||||
if (!fs.existsSync(annotationsDir)) {
|
||||
fs.mkdirSync(annotationsDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Write to files in the annotations directory
|
||||
const trainPath = `${annotationsDir}/coco_project_${trainingId}_train.json`;
|
||||
const validPath = `${annotationsDir}/coco_project_${trainingId}_valid.json`;
|
||||
const testPath = `${annotationsDir}/coco_project_${trainingId}_test.json`;
|
||||
fs.writeFileSync(trainPath, JSON.stringify(trainJson, null, 2));
|
||||
fs.writeFileSync(validPath, JSON.stringify(validJson, null, 2));
|
||||
fs.writeFileSync(testPath, JSON.stringify(testJson, null, 2));
|
||||
console.log(`COCO JSON splits written to ${annotationsDir} for trainingId ${trainingId}`);
|
||||
|
||||
|
||||
|
||||
// Also generate inference exp.py in the same output directory as exp.py (project folder in workspace)
|
||||
const { generateYoloxInferenceExp } = require('./generate-yolox-exp');
|
||||
const path = require('path');
|
||||
const projectFolder = path.join(__dirname, '..', projectName, String(trainingId));
|
||||
if (!fs.existsSync(projectFolder)) {
|
||||
fs.mkdirSync(projectFolder, { recursive: true });
|
||||
}
|
||||
const inferenceExpPath = path.join(projectFolder, 'exp_infer.py');
|
||||
generateYoloxInferenceExp(trainingId).then(expContent => {
|
||||
fs.writeFileSync(inferenceExpPath, expContent);
|
||||
console.log(`Inference exp.py written to ${inferenceExpPath}`);
|
||||
}).catch(err => {
|
||||
console.error('Failed to generate inference exp.py:', err);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
module.exports = {generateTrainingJson};
|
||||
@@ -1,135 +1,135 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Training = require('../models/training.js');
|
||||
const TrainingProject = require('../models/TrainingProject.js');
|
||||
|
||||
// Remove Python comments and legacy code
|
||||
const exp_names = [
|
||||
'YOLOX-s',
|
||||
'YOLOX-m',
|
||||
'YOLOX-l',
|
||||
'YOLOX-x',
|
||||
'YOLOX-Darknet53', //todo
|
||||
'YOLOX-Nano',
|
||||
'YOLOX-Tiny'
|
||||
]
|
||||
|
||||
//TODO: Clean up generation of exp_names.py and remove second exp creation!!!
|
||||
|
||||
|
||||
// Refactored: Accept trainingId, fetch info from DB
|
||||
async function generateYoloxExp(trainingId) {
|
||||
// Fetch training row from DB by project_details_id if not found by PK
|
||||
let training = await Training.findByPk(trainingId);
|
||||
if (!training) {
|
||||
training = await Training.findOne({ where: { project_details_id: trainingId } });
|
||||
}
|
||||
if (!training) throw new Error('Training not found for trainingId or project_details_id: ' + trainingId);
|
||||
|
||||
// If transfer_learning is 'coco', just return the path to the default exp.py
|
||||
if (training.transfer_learning === 'coco') {
|
||||
const selectedModel = training.selected_model.toLowerCase().replace('-', '_');
|
||||
const expSourcePath = `/home/kitraining/Yolox/YOLOX-main/exps/default/${selectedModel}.py`;
|
||||
if (!fs.existsSync(expSourcePath)) {
|
||||
throw new Error(`Default exp.py not found for model: ${selectedModel} at ${expSourcePath}`);
|
||||
}
|
||||
// Copy to project folder (e.g., /home/kitraining/coco_tool/backend/project_XX/YY/exp.py)
|
||||
const projectDetailsId = training.project_details_id;
|
||||
const projectFolder = path.resolve(__dirname, `../project_23/${projectDetailsId}`);
|
||||
if (!fs.existsSync(projectFolder)) {
|
||||
fs.mkdirSync(projectFolder, { recursive: true });
|
||||
}
|
||||
const expDestPath = path.join(projectFolder, 'exp.py');
|
||||
fs.copyFileSync(expSourcePath, expDestPath);
|
||||
return { type: 'default', expPath: expDestPath };
|
||||
}
|
||||
|
||||
// If transfer_learning is 'sketch', generate a custom exp.py as before
|
||||
if (training.transfer_learning === 'sketch') {
|
||||
// ...existing custom exp.py generation logic here (copy from previous implementation)...
|
||||
// For brevity, you can call generateYoloxInferenceExp or similar here, or inline the logic.
|
||||
// Example:
|
||||
const expContent = await generateYoloxInferenceExp(trainingId);
|
||||
return { type: 'custom', expContent };
|
||||
}
|
||||
|
||||
throw new Error('Unknown transfer_learning type: ' + training.transfer_learning);
|
||||
}
|
||||
|
||||
async function saveYoloxExp(trainingId, outPath) {
|
||||
const expResult = await generateYoloxExp(trainingId);
|
||||
if (expResult.type === 'custom' && expResult.expContent) {
|
||||
fs.writeFileSync(outPath, expResult.expContent);
|
||||
return outPath;
|
||||
} else if (expResult.type === 'default' && expResult.expPath) {
|
||||
// Optionally copy the file if outPath is different
|
||||
if (expResult.expPath !== outPath) {
|
||||
fs.copyFileSync(expResult.expPath, outPath);
|
||||
}
|
||||
return outPath;
|
||||
} else {
|
||||
throw new Error('Unknown expResult type or missing content');
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a second exp.py for inference, using the provided template and DB values
|
||||
async function generateYoloxInferenceExp(trainingId, options = {}) {
|
||||
let training = await Training.findByPk(trainingId);
|
||||
if (!training) {
|
||||
training = await Training.findOne({ where: { project_details_id: trainingId } });
|
||||
}
|
||||
if (!training) throw new Error('Training not found for trainingId or project_details_id: ' + trainingId);
|
||||
// Always use the trainingId (project_details_id) for annotation file names
|
||||
const projectDetailsId = training.project_details_id;
|
||||
const dataDir = options.data_dir || '/home/kitraining/To_Annotate/';
|
||||
const trainAnn = options.train_ann || `coco_project_${trainingId}_train.json`;
|
||||
const valAnn = options.val_ann || `coco_project_${trainingId}_valid.json`;
|
||||
const testAnn = options.test_ann || `coco_project_${trainingId}_test.json`;
|
||||
// Get num_classes from TrainingProject.classes JSON
|
||||
let numClasses = 80;
|
||||
try {
|
||||
const trainingProject = await TrainingProject.findByPk(projectDetailsId);
|
||||
if (trainingProject && trainingProject.classes) {
|
||||
let classesArr = trainingProject.classes;
|
||||
if (typeof classesArr === 'string') {
|
||||
classesArr = JSON.parse(classesArr);
|
||||
}
|
||||
if (Array.isArray(classesArr)) {
|
||||
numClasses = classesArr.filter(c => c !== null && c !== undefined && c !== '').length;
|
||||
} else if (typeof classesArr === 'object' && classesArr !== null) {
|
||||
numClasses = Object.keys(classesArr).filter(k => classesArr[k] !== null && classesArr[k] !== undefined && classesArr[k] !== '').length;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Could not determine num_classes from TrainingProject.classes:', e);
|
||||
}
|
||||
const depth = options.depth || training.depth || 1.00;
|
||||
const width = options.width || training.width || 1.00;
|
||||
const inputSize = options.input_size || training.input_size || [640, 640];
|
||||
const mosaicScale = options.mosaic_scale || training.mosaic_scale || [0.1, 2];
|
||||
const randomSize = options.random_size || training.random_size || [10, 20];
|
||||
const testSize = options.test_size || training.test_size || [640, 640];
|
||||
const expName = options.exp_name || 'inference_exp';
|
||||
const enableMixup = options.enable_mixup !== undefined ? options.enable_mixup : false;
|
||||
let expContent = '';
|
||||
expContent += `#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.data_dir = "${dataDir}"\n self.train_ann = "${trainAnn}"\n self.val_ann = "${valAnn}"\n self.test_ann = "coco_project_${trainingId}_test.json"\n self.num_classes = ${numClasses}\n`;
|
||||
// Set pretrained_ckpt if transfer_learning is 'coco'
|
||||
if (training.transfer_learning && typeof training.transfer_learning === 'string' && training.transfer_learning.toLowerCase() === 'coco') {
|
||||
const yoloxBaseDir = '/home/kitraining/Yolox/YOLOX-main';
|
||||
const selectedModel = training.selected_model ? training.selected_model.replace(/\.pth$/i, '') : '';
|
||||
if (selectedModel) {
|
||||
expContent += ` self.pretrained_ckpt = r'${yoloxBaseDir}/pretrained/${selectedModel}.pth'\n`;
|
||||
}
|
||||
}
|
||||
expContent += ` self.depth = ${depth}\n self.width = ${width}\n self.input_size = (${Array.isArray(inputSize) ? inputSize.join(', ') : inputSize})\n self.mosaic_scale = (${Array.isArray(mosaicScale) ? mosaicScale.join(', ') : mosaicScale})\n self.random_size = (${Array.isArray(randomSize) ? randomSize.join(', ') : randomSize})\n self.test_size = (${Array.isArray(testSize) ? testSize.join(', ') : testSize})\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]\n self.enable_mixup = ${enableMixup ? 'True' : 'False'}\n`;
|
||||
return expContent;
|
||||
}
|
||||
|
||||
// Save inference exp.py to a custom path
|
||||
async function saveYoloxInferenceExp(trainingId, outPath, options = {}) {
|
||||
const expContent = await generateYoloxInferenceExp(trainingId, options);
|
||||
fs.writeFileSync(outPath, expContent);
|
||||
return outPath;
|
||||
}
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Training = require('../models/training.js');
|
||||
const TrainingProject = require('../models/TrainingProject.js');
|
||||
|
||||
// Remove Python comments and legacy code
|
||||
const exp_names = [
|
||||
'YOLOX-s',
|
||||
'YOLOX-m',
|
||||
'YOLOX-l',
|
||||
'YOLOX-x',
|
||||
'YOLOX-Darknet53', //todo
|
||||
'YOLOX-Nano',
|
||||
'YOLOX-Tiny'
|
||||
]
|
||||
|
||||
//TODO: Clean up generation of exp_names.py and remove second exp creation!!!
|
||||
|
||||
|
||||
// Refactored: Accept trainingId, fetch info from DB
|
||||
async function generateYoloxExp(trainingId) {
|
||||
// Fetch training row from DB by project_details_id if not found by PK
|
||||
let training = await Training.findByPk(trainingId);
|
||||
if (!training) {
|
||||
training = await Training.findOne({ where: { project_details_id: trainingId } });
|
||||
}
|
||||
if (!training) throw new Error('Training not found for trainingId or project_details_id: ' + trainingId);
|
||||
|
||||
// If transfer_learning is 'coco', just return the path to the default exp.py
|
||||
if (training.transfer_learning === 'coco') {
|
||||
const selectedModel = training.selected_model.toLowerCase().replace('-', '_');
|
||||
const expSourcePath = `/home/kitraining/Yolox/YOLOX-main/exps/default/${selectedModel}.py`;
|
||||
if (!fs.existsSync(expSourcePath)) {
|
||||
throw new Error(`Default exp.py not found for model: ${selectedModel} at ${expSourcePath}`);
|
||||
}
|
||||
// Copy to project folder (e.g., /home/kitraining/coco_tool/backend/project_XX/YY/exp.py)
|
||||
const projectDetailsId = training.project_details_id;
|
||||
const projectFolder = path.resolve(__dirname, `../project_23/${projectDetailsId}`);
|
||||
if (!fs.existsSync(projectFolder)) {
|
||||
fs.mkdirSync(projectFolder, { recursive: true });
|
||||
}
|
||||
const expDestPath = path.join(projectFolder, 'exp.py');
|
||||
fs.copyFileSync(expSourcePath, expDestPath);
|
||||
return { type: 'default', expPath: expDestPath };
|
||||
}
|
||||
|
||||
// If transfer_learning is 'sketch', generate a custom exp.py as before
|
||||
if (training.transfer_learning === 'sketch') {
|
||||
// ...existing custom exp.py generation logic here (copy from previous implementation)...
|
||||
// For brevity, you can call generateYoloxInferenceExp or similar here, or inline the logic.
|
||||
// Example:
|
||||
const expContent = await generateYoloxInferenceExp(trainingId);
|
||||
return { type: 'custom', expContent };
|
||||
}
|
||||
|
||||
throw new Error('Unknown transfer_learning type: ' + training.transfer_learning);
|
||||
}
|
||||
|
||||
async function saveYoloxExp(trainingId, outPath) {
|
||||
const expResult = await generateYoloxExp(trainingId);
|
||||
if (expResult.type === 'custom' && expResult.expContent) {
|
||||
fs.writeFileSync(outPath, expResult.expContent);
|
||||
return outPath;
|
||||
} else if (expResult.type === 'default' && expResult.expPath) {
|
||||
// Optionally copy the file if outPath is different
|
||||
if (expResult.expPath !== outPath) {
|
||||
fs.copyFileSync(expResult.expPath, outPath);
|
||||
}
|
||||
return outPath;
|
||||
} else {
|
||||
throw new Error('Unknown expResult type or missing content');
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a second exp.py for inference, using the provided template and DB values
|
||||
async function generateYoloxInferenceExp(trainingId, options = {}) {
|
||||
let training = await Training.findByPk(trainingId);
|
||||
if (!training) {
|
||||
training = await Training.findOne({ where: { project_details_id: trainingId } });
|
||||
}
|
||||
if (!training) throw new Error('Training not found for trainingId or project_details_id: ' + trainingId);
|
||||
// Always use the trainingId (project_details_id) for annotation file names
|
||||
const projectDetailsId = training.project_details_id;
|
||||
const dataDir = options.data_dir || '/home/kitraining/To_Annotate/';
|
||||
const trainAnn = options.train_ann || `coco_project_${trainingId}_train.json`;
|
||||
const valAnn = options.val_ann || `coco_project_${trainingId}_valid.json`;
|
||||
const testAnn = options.test_ann || `coco_project_${trainingId}_test.json`;
|
||||
// Get num_classes from TrainingProject.classes JSON
|
||||
let numClasses = 80;
|
||||
try {
|
||||
const trainingProject = await TrainingProject.findByPk(projectDetailsId);
|
||||
if (trainingProject && trainingProject.classes) {
|
||||
let classesArr = trainingProject.classes;
|
||||
if (typeof classesArr === 'string') {
|
||||
classesArr = JSON.parse(classesArr);
|
||||
}
|
||||
if (Array.isArray(classesArr)) {
|
||||
numClasses = classesArr.filter(c => c !== null && c !== undefined && c !== '').length;
|
||||
} else if (typeof classesArr === 'object' && classesArr !== null) {
|
||||
numClasses = Object.keys(classesArr).filter(k => classesArr[k] !== null && classesArr[k] !== undefined && classesArr[k] !== '').length;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Could not determine num_classes from TrainingProject.classes:', e);
|
||||
}
|
||||
const depth = options.depth || training.depth || 1.00;
|
||||
const width = options.width || training.width || 1.00;
|
||||
const inputSize = options.input_size || training.input_size || [640, 640];
|
||||
const mosaicScale = options.mosaic_scale || training.mosaic_scale || [0.1, 2];
|
||||
const randomSize = options.random_size || training.random_size || [10, 20];
|
||||
const testSize = options.test_size || training.test_size || [640, 640];
|
||||
const expName = options.exp_name || 'inference_exp';
|
||||
const enableMixup = options.enable_mixup !== undefined ? options.enable_mixup : false;
|
||||
let expContent = '';
|
||||
expContent += `#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.data_dir = "${dataDir}"\n self.train_ann = "${trainAnn}"\n self.val_ann = "${valAnn}"\n self.test_ann = "coco_project_${trainingId}_test.json"\n self.num_classes = ${numClasses}\n`;
|
||||
// Set pretrained_ckpt if transfer_learning is 'coco'
|
||||
if (training.transfer_learning && typeof training.transfer_learning === 'string' && training.transfer_learning.toLowerCase() === 'coco') {
|
||||
const yoloxBaseDir = '/home/kitraining/Yolox/YOLOX-main';
|
||||
const selectedModel = training.selected_model ? training.selected_model.replace(/\.pth$/i, '') : '';
|
||||
if (selectedModel) {
|
||||
expContent += ` self.pretrained_ckpt = r'${yoloxBaseDir}/pretrained/${selectedModel}.pth'\n`;
|
||||
}
|
||||
}
|
||||
expContent += ` self.depth = ${depth}\n self.width = ${width}\n self.input_size = (${Array.isArray(inputSize) ? inputSize.join(', ') : inputSize})\n self.mosaic_scale = (${Array.isArray(mosaicScale) ? mosaicScale.join(', ') : mosaicScale})\n self.random_size = (${Array.isArray(randomSize) ? randomSize.join(', ') : randomSize})\n self.test_size = (${Array.isArray(testSize) ? testSize.join(', ') : testSize})\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]\n self.enable_mixup = ${enableMixup ? 'True' : 'False'}\n`;
|
||||
return expContent;
|
||||
}
|
||||
|
||||
// Save inference exp.py to a custom path
|
||||
async function saveYoloxInferenceExp(trainingId, outPath, options = {}) {
|
||||
const expContent = await generateYoloxInferenceExp(trainingId, options);
|
||||
fs.writeFileSync(outPath, expContent);
|
||||
return outPath;
|
||||
}
|
||||
|
||||
module.exports = { generateYoloxExp, saveYoloxExp, generateYoloxInferenceExp, saveYoloxInferenceExp };
|
||||
@@ -1,179 +1,288 @@
|
||||
import json
|
||||
import os
|
||||
import math
|
||||
from models.TrainingProject import TrainingProject
|
||||
from models.TrainingProjectDetails import TrainingProjectDetails
|
||||
from models.Images import Image
|
||||
from models.Annotation import Annotation
|
||||
|
||||
def generate_training_json(training_id):
|
||||
"""Generate COCO JSON for training, validation, and test sets"""
|
||||
# training_id is now project_details_id
|
||||
training_project_details = TrainingProjectDetails.query.get(training_id)
|
||||
|
||||
if not training_project_details:
|
||||
raise Exception(f'No TrainingProjectDetails found for project_details_id {training_id}')
|
||||
|
||||
details_obj = training_project_details.to_dict()
|
||||
|
||||
# Get parent project for name
|
||||
training_project = TrainingProject.query.get(details_obj['project_id'])
|
||||
|
||||
# Get split percentages (default values if not set)
|
||||
train_percent = details_obj.get('train_percent', 85)
|
||||
valid_percent = details_obj.get('valid_percent', 10)
|
||||
test_percent = details_obj.get('test_percent', 5)
|
||||
|
||||
coco_images = []
|
||||
coco_annotations = []
|
||||
coco_categories = []
|
||||
category_map = {}
|
||||
category_id = 0
|
||||
image_id = 0
|
||||
annotation_id = 0
|
||||
|
||||
for cls in details_obj['class_map']:
|
||||
asg_map = []
|
||||
list_asg = cls[1]
|
||||
|
||||
for asg in list_asg:
|
||||
asg_map.append({'original': asg[0], 'mapped': asg[1]})
|
||||
# Build category list and mapping
|
||||
if asg[1] and asg[1] not in category_map:
|
||||
category_map[asg[1]] = category_id
|
||||
coco_categories.append({'id': category_id, 'name': asg[1], 'supercategory': ''})
|
||||
category_id += 1
|
||||
|
||||
# Get images for this project
|
||||
images = Image.query.filter_by(project_id=cls[0]).all()
|
||||
|
||||
for image in images:
|
||||
image_id += 1
|
||||
file_name = image.image_path
|
||||
|
||||
# Clean up file path
|
||||
if '%20' in file_name:
|
||||
file_name = file_name.replace('%20', ' ')
|
||||
if file_name and file_name.startswith('/data/local-files/?d='):
|
||||
file_name = file_name.replace('/data/local-files/?d=', '')
|
||||
file_name = file_name.replace('/home/kitraining/home/kitraining/', '')
|
||||
if file_name and file_name.startswith('home/kitraining/To_Annotate/'):
|
||||
file_name = file_name.replace('home/kitraining/To_Annotate/', '')
|
||||
|
||||
# Get annotations for this image
|
||||
annotations = Annotation.query.filter_by(image_id=image.image_id).all()
|
||||
|
||||
coco_images.append({
|
||||
'id': image_id,
|
||||
'file_name': file_name,
|
||||
'width': image.width or 0,
|
||||
'height': image.height or 0
|
||||
})
|
||||
|
||||
for annotation in annotations:
|
||||
# Translate class name using asg_map
|
||||
mapped_class = annotation.Label
|
||||
for map_entry in asg_map:
|
||||
if annotation.Label == map_entry['original']:
|
||||
mapped_class = map_entry['mapped']
|
||||
break
|
||||
|
||||
# Only add annotation if mapped_class is valid
|
||||
if mapped_class and mapped_class in category_map:
|
||||
annotation_id += 1
|
||||
area = 0
|
||||
if annotation.width and annotation.height:
|
||||
area = annotation.width * annotation.height
|
||||
|
||||
coco_annotations.append({
|
||||
'id': annotation_id,
|
||||
'image_id': image_id,
|
||||
'category_id': category_map[mapped_class],
|
||||
'bbox': [annotation.x, annotation.y, annotation.width, annotation.height],
|
||||
'area': area,
|
||||
'iscrowd': 0
|
||||
})
|
||||
|
||||
# Shuffle images for random split using seed
|
||||
def seeded_random(seed):
|
||||
x = math.sin(seed) * 10000
|
||||
return x - math.floor(x)
|
||||
|
||||
def shuffle(array, seed):
|
||||
for i in range(len(array) - 1, 0, -1):
|
||||
j = int(seeded_random(seed + i) * (i + 1))
|
||||
array[i], array[j] = array[j], array[i]
|
||||
|
||||
# Use seed from details_obj if present, else default to 42
|
||||
split_seed = details_obj.get('seed', 42)
|
||||
if split_seed is not None:
|
||||
split_seed = int(split_seed)
|
||||
else:
|
||||
split_seed = 42
|
||||
|
||||
shuffle(coco_images, split_seed)
|
||||
|
||||
# Split images
|
||||
total_images = len(coco_images)
|
||||
train_count = int(total_images * train_percent / 100)
|
||||
valid_count = int(total_images * valid_percent / 100)
|
||||
test_count = total_images - train_count - valid_count
|
||||
|
||||
train_images = coco_images[0:train_count]
|
||||
valid_images = coco_images[train_count:train_count + valid_count]
|
||||
test_images = coco_images[train_count + valid_count:]
|
||||
|
||||
# Helper to get image ids for each split
|
||||
train_image_ids = {img['id'] for img in train_images}
|
||||
valid_image_ids = {img['id'] for img in valid_images}
|
||||
test_image_ids = {img['id'] for img in test_images}
|
||||
|
||||
# Split annotations
|
||||
train_annotations = [ann for ann in coco_annotations if ann['image_id'] in train_image_ids]
|
||||
valid_annotations = [ann for ann in coco_annotations if ann['image_id'] in valid_image_ids]
|
||||
test_annotations = [ann for ann in coco_annotations if ann['image_id'] in test_image_ids]
|
||||
|
||||
# Build final COCO JSONs
|
||||
def build_coco_json(images, annotations, categories):
|
||||
return {
|
||||
'images': images,
|
||||
'annotations': annotations,
|
||||
'categories': categories
|
||||
}
|
||||
|
||||
train_json = build_coco_json(train_images, train_annotations, coco_categories)
|
||||
valid_json = build_coco_json(valid_images, valid_annotations, coco_categories)
|
||||
test_json = build_coco_json(test_images, test_annotations, coco_categories)
|
||||
|
||||
# Create output directory
|
||||
project_name = training_project.title.replace(' ', '_') if training_project and training_project.title else f'project_{details_obj["project_id"]}'
|
||||
annotations_dir = '/home/kitraining/To_Annotate/annotations'
|
||||
os.makedirs(annotations_dir, exist_ok=True)
|
||||
|
||||
# Write to files
|
||||
train_path = f'{annotations_dir}/coco_project_{training_id}_train.json'
|
||||
valid_path = f'{annotations_dir}/coco_project_{training_id}_valid.json'
|
||||
test_path = f'{annotations_dir}/coco_project_{training_id}_test.json'
|
||||
|
||||
with open(train_path, 'w') as f:
|
||||
json.dump(train_json, f, indent=2)
|
||||
with open(valid_path, 'w') as f:
|
||||
json.dump(valid_json, f, indent=2)
|
||||
with open(test_path, 'w') as f:
|
||||
json.dump(test_json, f, indent=2)
|
||||
|
||||
print(f'COCO JSON splits written to {annotations_dir} for trainingId {training_id}')
|
||||
|
||||
# Also generate inference exp.py
|
||||
from services.generate_yolox_exp import generate_yolox_inference_exp
|
||||
project_folder = os.path.join(os.path.dirname(__file__), '..', project_name, str(training_id))
|
||||
os.makedirs(project_folder, exist_ok=True)
|
||||
|
||||
inference_exp_path = os.path.join(project_folder, 'exp_infer.py')
|
||||
try:
|
||||
exp_content = generate_yolox_inference_exp(training_id)
|
||||
with open(inference_exp_path, 'w') as f:
|
||||
f.write(exp_content)
|
||||
print(f'Inference exp.py written to {inference_exp_path}')
|
||||
except Exception as err:
|
||||
print(f'Failed to generate inference exp.py: {err}')
|
||||
import json
|
||||
import os
|
||||
import math
|
||||
from models.TrainingProject import TrainingProject
|
||||
from models.TrainingProjectDetails import TrainingProjectDetails
|
||||
from models.Images import Image
|
||||
from models.Annotation import Annotation
|
||||
|
||||
def generate_training_json(training_id):
|
||||
"""Generate COCO JSON for training, validation, and test sets"""
|
||||
# training_id is now project_details_id
|
||||
training_project_details = TrainingProjectDetails.query.get(training_id)
|
||||
|
||||
if not training_project_details:
|
||||
raise Exception(f'No TrainingProjectDetails found for project_details_id {training_id}')
|
||||
|
||||
details_obj = training_project_details.to_dict()
|
||||
|
||||
# Get parent project for name
|
||||
training_project = TrainingProject.query.get(details_obj['project_id'])
|
||||
|
||||
# Get the data directory setting for image paths
|
||||
from services.settings_service import get_setting
|
||||
data_dir = get_setting('yolox_data_dir', '/home/kitraining/To_Annotate/')
|
||||
|
||||
# Fix UNC path if it's missing the \\ prefix
|
||||
# Check if it looks like a UNC path without proper prefix (e.g., "192.168.1.19\...")
|
||||
if data_dir and not data_dir.startswith('\\\\') and not data_dir.startswith('/'):
|
||||
# Check if it starts with an IP address pattern
|
||||
import re
|
||||
if re.match(r'^\d+\.\d+\.\d+\.\d+[/\\]', data_dir):
|
||||
data_dir = '\\\\' + data_dir
|
||||
|
||||
# Ensure data_dir ends with separator
|
||||
if not data_dir.endswith(os.sep) and not data_dir.endswith('/'):
|
||||
data_dir += os.sep
|
||||
|
||||
# Get split percentages (default values if not set)
|
||||
train_percent = details_obj.get('train_percent', 85)
|
||||
valid_percent = details_obj.get('valid_percent', 10)
|
||||
test_percent = details_obj.get('test_percent', 5)
|
||||
|
||||
coco_images = []
|
||||
coco_annotations = []
|
||||
coco_categories = []
|
||||
category_map = {}
|
||||
category_id = 0
|
||||
image_id = 0
|
||||
annotation_id = 0
|
||||
|
||||
# Build category list and mapping from class_map dictionary {source: target}
|
||||
class_map = details_obj.get('class_map', {})
|
||||
|
||||
for source_class, target_class in class_map.items():
|
||||
if target_class and target_class not in category_map:
|
||||
category_map[target_class] = category_id
|
||||
coco_categories.append({'id': category_id, 'name': target_class, 'supercategory': ''})
|
||||
category_id += 1
|
||||
|
||||
# Get all annotation projects (Label Studio project IDs)
|
||||
annotation_projects = details_obj.get('annotation_projects', [])
|
||||
|
||||
# Get class mappings from database grouped by Label Studio project
|
||||
from models.ClassMapping import ClassMapping
|
||||
all_mappings = ClassMapping.query.filter_by(project_details_id=training_id).all()
|
||||
|
||||
# Group mappings by Label Studio project ID
|
||||
mappings_by_project = {}
|
||||
for mapping in all_mappings:
|
||||
ls_proj_id = mapping.label_studio_project_id
|
||||
if ls_proj_id not in mappings_by_project:
|
||||
mappings_by_project[ls_proj_id] = {}
|
||||
mappings_by_project[ls_proj_id][mapping.source_class] = mapping.target_class
|
||||
|
||||
# Also add target class to category map if not present
|
||||
if mapping.target_class and mapping.target_class not in category_map:
|
||||
category_map[mapping.target_class] = category_id
|
||||
coco_categories.append({'id': category_id, 'name': mapping.target_class, 'supercategory': ''})
|
||||
category_id += 1
|
||||
|
||||
# Iterate through each annotation project to collect images and annotations
|
||||
for ls_project_id in annotation_projects:
|
||||
# Get images for this Label Studio project
|
||||
images = Image.query.filter_by(project_id=ls_project_id).all()
|
||||
|
||||
for image in images:
|
||||
image_id += 1
|
||||
file_name = image.image_path
|
||||
|
||||
# Clean up file path from Label Studio format
|
||||
if '%20' in file_name:
|
||||
file_name = file_name.replace('%20', ' ')
|
||||
if file_name and file_name.startswith('/data/local-files/?d='):
|
||||
file_name = file_name.replace('/data/local-files/?d=', '')
|
||||
|
||||
# Remove any Label Studio prefixes but keep full path
|
||||
# Common Label Studio patterns
|
||||
prefixes_to_remove = [
|
||||
'//192.168.1.19/home/kitraining/To_Annotate/',
|
||||
'192.168.1.19/home/kitraining/To_Annotate/',
|
||||
'/home/kitraining/home/kitraining/',
|
||||
'home/kitraining/To_Annotate/',
|
||||
'/home/kitraining/To_Annotate/',
|
||||
]
|
||||
|
||||
# Try each prefix
|
||||
for prefix in prefixes_to_remove:
|
||||
if file_name.startswith(prefix):
|
||||
file_name = file_name[len(prefix):]
|
||||
break
|
||||
|
||||
# Construct ABSOLUTE path using data_dir
|
||||
# Detect platform for proper path handling
|
||||
import platform
|
||||
is_windows = platform.system() == 'Windows'
|
||||
|
||||
# Normalize path separators in file_name to forward slashes first (OS-agnostic)
|
||||
file_name = file_name.replace('\\', '/')
|
||||
|
||||
# Normalize data_dir to use forward slashes
|
||||
normalized_data_dir = data_dir.rstrip('/\\').replace('\\', '/')
|
||||
|
||||
# Check if file_name is already an absolute path
|
||||
is_absolute = False
|
||||
if is_windows:
|
||||
# Windows: Check for drive letter (C:/) or UNC path (//server/)
|
||||
is_absolute = (len(file_name) > 1 and file_name[1] == ':') or file_name.startswith('//')
|
||||
else:
|
||||
# Linux/Mac: Check for leading /
|
||||
is_absolute = file_name.startswith('/')
|
||||
|
||||
if not is_absolute:
|
||||
# It's a relative path, combine with data_dir
|
||||
if normalized_data_dir.startswith('//'):
|
||||
# UNC path on Windows
|
||||
file_name = normalized_data_dir + '/' + file_name
|
||||
else:
|
||||
# Regular path - use os.path.join but with forward slashes
|
||||
file_name = os.path.join(normalized_data_dir, file_name).replace('\\', '/')
|
||||
|
||||
# Final OS-specific normalization
|
||||
if is_windows:
|
||||
# Convert to Windows-style backslashes
|
||||
file_name = file_name.replace('/', '\\')
|
||||
else:
|
||||
# Keep as forward slashes for Linux/Mac
|
||||
file_name = file_name.replace('\\', '/')
|
||||
|
||||
# Get annotations for this image
|
||||
annotations = Annotation.query.filter_by(image_id=image.image_id).all()
|
||||
|
||||
# Ensure width and height are integers and valid
|
||||
# If missing or invalid, skip this image or use default dimensions
|
||||
img_width = int(image.width) if image.width else 0
|
||||
img_height = int(image.height) if image.height else 0
|
||||
|
||||
# Skip images with invalid dimensions
|
||||
if img_width <= 0 or img_height <= 0:
|
||||
print(f'Warning: Skipping image {file_name} with invalid dimensions: {img_width}x{img_height}')
|
||||
continue
|
||||
|
||||
coco_images.append({
|
||||
'id': image_id,
|
||||
'file_name': file_name, # Use absolute path
|
||||
'width': img_width,
|
||||
'height': img_height
|
||||
})
|
||||
|
||||
for annotation in annotations:
|
||||
# Translate class name using class_map for this specific Label Studio project
|
||||
original_class = annotation.Label
|
||||
project_class_map = mappings_by_project.get(ls_project_id, {})
|
||||
mapped_class = project_class_map.get(original_class, original_class)
|
||||
|
||||
# Only add annotation if mapped_class is valid
|
||||
if mapped_class and mapped_class in category_map:
|
||||
annotation_id += 1
|
||||
area = 0
|
||||
if annotation.width and annotation.height:
|
||||
area = annotation.width * annotation.height
|
||||
|
||||
coco_annotations.append({
|
||||
'id': annotation_id,
|
||||
'image_id': image_id,
|
||||
'category_id': category_map[mapped_class],
|
||||
'bbox': [annotation.x, annotation.y, annotation.width, annotation.height],
|
||||
'area': area,
|
||||
'iscrowd': 0
|
||||
})
|
||||
|
||||
# Shuffle images for random split using seed
|
||||
def seeded_random(seed):
|
||||
x = math.sin(seed) * 10000
|
||||
return x - math.floor(x)
|
||||
|
||||
def shuffle(array, seed):
|
||||
for i in range(len(array) - 1, 0, -1):
|
||||
j = int(seeded_random(seed + i) * (i + 1))
|
||||
array[i], array[j] = array[j], array[i]
|
||||
|
||||
# Use seed from details_obj if present, else default to 42
|
||||
split_seed = details_obj.get('seed', 42)
|
||||
if split_seed is not None:
|
||||
split_seed = int(split_seed)
|
||||
else:
|
||||
split_seed = 42
|
||||
|
||||
shuffle(coco_images, split_seed)
|
||||
|
||||
# Split images
|
||||
total_images = len(coco_images)
|
||||
train_count = int(total_images * train_percent / 100)
|
||||
valid_count = int(total_images * valid_percent / 100)
|
||||
test_count = total_images - train_count - valid_count
|
||||
|
||||
train_images = coco_images[0:train_count]
|
||||
valid_images = coco_images[train_count:train_count + valid_count]
|
||||
test_images = coco_images[train_count + valid_count:]
|
||||
|
||||
# Helper to get image ids for each split
|
||||
train_image_ids = {img['id'] for img in train_images}
|
||||
valid_image_ids = {img['id'] for img in valid_images}
|
||||
test_image_ids = {img['id'] for img in test_images}
|
||||
|
||||
# Split annotations
|
||||
train_annotations = [ann for ann in coco_annotations if ann['image_id'] in train_image_ids]
|
||||
valid_annotations = [ann for ann in coco_annotations if ann['image_id'] in valid_image_ids]
|
||||
test_annotations = [ann for ann in coco_annotations if ann['image_id'] in test_image_ids]
|
||||
|
||||
# Build final COCO JSONs
|
||||
def build_coco_json(images, annotations, categories):
|
||||
return {
|
||||
'images': images,
|
||||
'annotations': annotations,
|
||||
'categories': categories
|
||||
}
|
||||
|
||||
train_json = build_coco_json(train_images, train_annotations, coco_categories)
|
||||
valid_json = build_coco_json(valid_images, valid_annotations, coco_categories)
|
||||
test_json = build_coco_json(test_images, test_annotations, coco_categories)
|
||||
|
||||
# Create output directory
|
||||
from services.settings_service import get_setting
|
||||
from models.training import Training
|
||||
|
||||
output_base_path = get_setting('yolox_output_path', './backend')
|
||||
|
||||
project_name = training_project.title.replace(' ', '_') if training_project and training_project.title else f'project_{details_obj["project_id"]}'
|
||||
|
||||
# Get training record to use its name for folder
|
||||
training_record = Training.query.filter_by(project_details_id=training_id).first()
|
||||
training_folder_name = f"{training_record.exp_name or training_record.training_name or 'training'}_{training_record.id}" if training_record else str(training_id)
|
||||
training_folder_name = training_folder_name.replace(' ', '_')
|
||||
|
||||
# Use training_record.id for file names to match what generate_yolox_exp expects
|
||||
training_file_id = training_record.id if training_record else training_id
|
||||
|
||||
# Save annotations to the configured output folder
|
||||
annotations_dir = os.path.join(output_base_path, project_name, training_folder_name, 'annotations')
|
||||
os.makedirs(annotations_dir, exist_ok=True)
|
||||
|
||||
# Write to files
|
||||
train_path = os.path.join(annotations_dir, f'coco_project_{training_file_id}_train.json')
|
||||
valid_path = os.path.join(annotations_dir, f'coco_project_{training_file_id}_valid.json')
|
||||
test_path = os.path.join(annotations_dir, f'coco_project_{training_file_id}_test.json')
|
||||
|
||||
with open(train_path, 'w') as f:
|
||||
json.dump(train_json, f, indent=2)
|
||||
with open(valid_path, 'w') as f:
|
||||
json.dump(valid_json, f, indent=2)
|
||||
with open(test_path, 'w') as f:
|
||||
json.dump(test_json, f, indent=2)
|
||||
|
||||
print(f'COCO JSON splits written to {annotations_dir} for trainingId {training_id}')
|
||||
|
||||
# Also generate inference exp.py
|
||||
from services.generate_yolox_exp import generate_yolox_inference_exp
|
||||
project_folder = os.path.join(output_base_path, project_name, str(training_id))
|
||||
os.makedirs(project_folder, exist_ok=True)
|
||||
|
||||
inference_exp_path = os.path.join(project_folder, 'exp_infer.py')
|
||||
try:
|
||||
exp_content = generate_yolox_inference_exp(training_id)
|
||||
with open(inference_exp_path, 'w') as f:
|
||||
f.write(exp_content)
|
||||
print(f'Inference exp.py written to {inference_exp_path}')
|
||||
except Exception as err:
|
||||
print(f'Failed to generate inference exp.py: {err}')
|
||||
|
||||
@@ -1,228 +1,329 @@
|
||||
import os
|
||||
import shutil
|
||||
import importlib.util
|
||||
from models.training import Training
|
||||
from models.TrainingProject import TrainingProject
|
||||
|
||||
def load_base_config(selected_model):
|
||||
"""Load base configuration for a specific YOLOX model"""
|
||||
model_name = selected_model.lower().replace('-', '_').replace('.pth', '')
|
||||
base_config_path = os.path.join(os.path.dirname(__file__), '..', 'data', f'{model_name}.py')
|
||||
|
||||
if not os.path.exists(base_config_path):
|
||||
raise Exception(f'Base configuration not found for model: {model_name} at {base_config_path}')
|
||||
|
||||
# Load the module dynamically
|
||||
spec = importlib.util.spec_from_file_location(f"base_config_{model_name}", base_config_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
# Extract all attributes from BaseExp class
|
||||
base_exp = module.BaseExp()
|
||||
base_config = {}
|
||||
for attr in dir(base_exp):
|
||||
if not attr.startswith('_'):
|
||||
base_config[attr] = getattr(base_exp, attr)
|
||||
|
||||
return base_config
|
||||
|
||||
def generate_yolox_exp(training_id):
|
||||
"""Generate YOLOX exp.py file"""
|
||||
# Fetch training row from DB
|
||||
training = Training.query.get(training_id)
|
||||
if not training:
|
||||
training = Training.query.filter_by(project_details_id=training_id).first()
|
||||
|
||||
if not training:
|
||||
raise Exception(f'Training not found for trainingId or project_details_id: {training_id}')
|
||||
|
||||
# If transfer_learning is 'coco', generate exp using base config + custom settings
|
||||
if training.transfer_learning == 'coco':
|
||||
exp_content = generate_yolox_inference_exp(training_id, use_base_config=True)
|
||||
return {'type': 'custom', 'expContent': exp_content}
|
||||
|
||||
# If transfer_learning is 'sketch', generate custom exp.py
|
||||
if training.transfer_learning == 'sketch':
|
||||
exp_content = generate_yolox_inference_exp(training_id, use_base_config=False)
|
||||
return {'type': 'custom', 'expContent': exp_content}
|
||||
|
||||
raise Exception(f'Unknown transfer_learning type: {training.transfer_learning}')
|
||||
|
||||
def save_yolox_exp(training_id, out_path):
|
||||
"""Save YOLOX exp.py to specified path"""
|
||||
exp_result = generate_yolox_exp(training_id)
|
||||
|
||||
if exp_result['type'] == 'custom' and 'expContent' in exp_result:
|
||||
with open(out_path, 'w') as f:
|
||||
f.write(exp_result['expContent'])
|
||||
return out_path
|
||||
elif exp_result['type'] == 'default' and 'expPath' in exp_result:
|
||||
# Optionally copy the file if outPath is different
|
||||
if exp_result['expPath'] != out_path:
|
||||
shutil.copyfile(exp_result['expPath'], out_path)
|
||||
return out_path
|
||||
else:
|
||||
raise Exception('Unknown expResult type or missing content')
|
||||
|
||||
def generate_yolox_inference_exp(training_id, options=None, use_base_config=False):
|
||||
"""Generate inference exp.py using DB values
|
||||
|
||||
Args:
|
||||
training_id: The training/project_details ID
|
||||
options: Optional overrides for data paths
|
||||
use_base_config: If True, load base config and only override with user-defined values
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
training = Training.query.get(training_id)
|
||||
if not training:
|
||||
training = Training.query.filter_by(project_details_id=training_id).first()
|
||||
|
||||
if not training:
|
||||
raise Exception(f'Training not found for trainingId or project_details_id: {training_id}')
|
||||
|
||||
# Always use the training_id (project_details_id) for annotation file names
|
||||
project_details_id = training.project_details_id
|
||||
|
||||
data_dir = options.get('data_dir', '/home/kitraining/To_Annotate/')
|
||||
train_ann = options.get('train_ann', f'coco_project_{training_id}_train.json')
|
||||
val_ann = options.get('val_ann', f'coco_project_{training_id}_valid.json')
|
||||
test_ann = options.get('test_ann', f'coco_project_{training_id}_test.json')
|
||||
|
||||
# Get num_classes from TrainingProject.classes JSON
|
||||
num_classes = 80
|
||||
try:
|
||||
training_project = TrainingProject.query.get(project_details_id)
|
||||
if training_project and training_project.classes:
|
||||
classes_arr = training_project.classes
|
||||
if isinstance(classes_arr, str):
|
||||
import json
|
||||
classes_arr = json.loads(classes_arr)
|
||||
|
||||
if isinstance(classes_arr, list):
|
||||
num_classes = len([c for c in classes_arr if c not in [None, '']])
|
||||
elif isinstance(classes_arr, dict):
|
||||
num_classes = len([k for k, v in classes_arr.items() if v not in [None, '']])
|
||||
except Exception as e:
|
||||
print(f'Could not determine num_classes from TrainingProject.classes: {e}')
|
||||
|
||||
# Initialize config dictionary
|
||||
config = {}
|
||||
|
||||
# If using base config (transfer learning from COCO), load protected parameters first
|
||||
if use_base_config and training.selected_model:
|
||||
try:
|
||||
base_config = load_base_config(training.selected_model)
|
||||
config.update(base_config)
|
||||
print(f'Loaded base config for {training.selected_model}: {list(base_config.keys())}')
|
||||
except Exception as e:
|
||||
print(f'Warning: Could not load base config for {training.selected_model}: {e}')
|
||||
print('Falling back to custom settings only')
|
||||
|
||||
# Override with user-defined values from training table (only if they exist and are not None)
|
||||
user_overrides = {
|
||||
'depth': training.depth,
|
||||
'width': training.width,
|
||||
'input_size': training.input_size,
|
||||
'mosaic_scale': training.mosaic_scale,
|
||||
'test_size': training.test_size,
|
||||
'enable_mixup': training.enable_mixup,
|
||||
'max_epoch': training.max_epoch,
|
||||
'warmup_epochs': training.warmup_epochs,
|
||||
'warmup_lr': training.warmup_lr,
|
||||
'basic_lr_per_img': training.basic_lr_per_img,
|
||||
'scheduler': training.scheduler,
|
||||
'no_aug_epochs': training.no_aug_epochs,
|
||||
'min_lr_ratio': training.min_lr_ratio,
|
||||
'ema': training.ema,
|
||||
'weight_decay': training.weight_decay,
|
||||
'momentum': training.momentum,
|
||||
'print_interval': training.print_interval,
|
||||
'eval_interval': training.eval_interval,
|
||||
'test_conf': training.test_conf,
|
||||
'nms_thre': training.nms_thre,
|
||||
'mosaic_prob': training.mosaic_prob,
|
||||
'mixup_prob': training.mixup_prob,
|
||||
'hsv_prob': training.hsv_prob,
|
||||
'flip_prob': training.flip_prob,
|
||||
'degrees': training.degrees,
|
||||
'translate': training.translate,
|
||||
'shear': training.shear,
|
||||
'mixup_scale': training.mixup_scale,
|
||||
'activation': training.activation,
|
||||
}
|
||||
|
||||
# Only override if value is explicitly set (not None)
|
||||
for key, value in user_overrides.items():
|
||||
if value is not None:
|
||||
config[key] = value
|
||||
|
||||
# Apply any additional options overrides
|
||||
config.update(options)
|
||||
|
||||
# Set defaults for any missing required parameters
|
||||
config.setdefault('depth', 1.00)
|
||||
config.setdefault('width', 1.00)
|
||||
config.setdefault('input_size', [640, 640])
|
||||
config.setdefault('mosaic_scale', [0.1, 2])
|
||||
config.setdefault('random_size', [10, 20])
|
||||
config.setdefault('test_size', [640, 640])
|
||||
config.setdefault('enable_mixup', False)
|
||||
config.setdefault('exp_name', 'inference_exp')
|
||||
|
||||
# Build exp content
|
||||
exp_content = f'''#!/usr/bin/env python3
|
||||
# -*- coding:utf-8 -*-
|
||||
# Copyright (c) Megvii, Inc. and its affiliates.
|
||||
|
||||
import os
|
||||
|
||||
from yolox.exp import Exp as MyExp
|
||||
|
||||
|
||||
class Exp(MyExp):
|
||||
def __init__(self):
|
||||
super(Exp, self).__init__()
|
||||
self.data_dir = "{data_dir}"
|
||||
self.train_ann = "{train_ann}"
|
||||
self.val_ann = "{val_ann}"
|
||||
self.test_ann = "{test_ann}"
|
||||
self.num_classes = {num_classes}
|
||||
'''
|
||||
|
||||
# Set pretrained_ckpt if transfer_learning is 'coco'
|
||||
if training.transfer_learning and isinstance(training.transfer_learning, str) and training.transfer_learning.lower() == 'coco':
|
||||
yolox_base_dir = '/home/kitraining/Yolox/YOLOX-main'
|
||||
selected_model = training.selected_model.replace('.pth', '') if training.selected_model else ''
|
||||
if selected_model:
|
||||
exp_content += f" self.pretrained_ckpt = r'{yolox_base_dir}/pretrained/{selected_model}.pth'\n"
|
||||
|
||||
# Format arrays
|
||||
def format_value(val):
|
||||
if isinstance(val, (list, tuple)):
|
||||
return '(' + ', '.join(map(str, val)) + ')'
|
||||
elif isinstance(val, bool):
|
||||
return str(val)
|
||||
elif isinstance(val, str):
|
||||
return f'"{val}"'
|
||||
else:
|
||||
return str(val)
|
||||
|
||||
# Add all config parameters to exp
|
||||
for key, value in config.items():
|
||||
if key not in ['exp_name']: # exp_name is handled separately
|
||||
exp_content += f" self.{key} = {format_value(value)}\n"
|
||||
|
||||
# Add exp_name at the end (uses dynamic path)
|
||||
exp_content += f''' self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]
|
||||
'''
|
||||
|
||||
return exp_content
|
||||
|
||||
def save_yolox_inference_exp(training_id, out_path, options=None):
|
||||
"""Save inference exp.py to custom path"""
|
||||
exp_content = generate_yolox_inference_exp(training_id, options, use_base_config=False)
|
||||
with open(out_path, 'w') as f:
|
||||
f.write(exp_content)
|
||||
return out_path
|
||||
import os
|
||||
import shutil
|
||||
import importlib.util
|
||||
from models.training import Training
|
||||
from models.TrainingProject import TrainingProject
|
||||
|
||||
def load_base_config(selected_model):
|
||||
"""Load base configuration for a specific YOLOX model"""
|
||||
model_name = selected_model.lower().replace('-', '_').replace('.pth', '')
|
||||
base_config_path = os.path.join(os.path.dirname(__file__), '..', 'data', f'{model_name}.py')
|
||||
|
||||
if not os.path.exists(base_config_path):
|
||||
raise Exception(f'Base configuration not found for model: {model_name} at {base_config_path}')
|
||||
|
||||
# Load the module dynamically
|
||||
spec = importlib.util.spec_from_file_location(f"base_config_{model_name}", base_config_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
# Extract all attributes from BaseExp class
|
||||
base_exp = module.BaseExp()
|
||||
base_config = {}
|
||||
for attr in dir(base_exp):
|
||||
if not attr.startswith('_'):
|
||||
base_config[attr] = getattr(base_exp, attr)
|
||||
|
||||
return base_config
|
||||
|
||||
def generate_yolox_exp(training_id):
|
||||
"""Generate YOLOX exp.py file"""
|
||||
# Fetch training row from DB
|
||||
training = Training.query.get(training_id)
|
||||
if not training:
|
||||
training = Training.query.filter_by(project_details_id=training_id).first()
|
||||
|
||||
if not training:
|
||||
raise Exception(f'Training not found for trainingId or project_details_id: {training_id}')
|
||||
|
||||
# If transfer_learning is 'coco', generate exp using base config + custom settings
|
||||
if training.transfer_learning == 'coco':
|
||||
exp_content = generate_yolox_inference_exp(training_id, use_base_config=True)
|
||||
return {'type': 'custom', 'expContent': exp_content}
|
||||
|
||||
# If transfer_learning is 'sketch', generate custom exp.py
|
||||
if training.transfer_learning == 'sketch':
|
||||
exp_content = generate_yolox_inference_exp(training_id, use_base_config=False)
|
||||
return {'type': 'custom', 'expContent': exp_content}
|
||||
|
||||
raise Exception(f'Unknown transfer_learning type: {training.transfer_learning}')
|
||||
|
||||
def save_yolox_exp(training_id, out_path):
|
||||
"""Save YOLOX exp.py to specified path"""
|
||||
exp_result = generate_yolox_exp(training_id)
|
||||
|
||||
if exp_result['type'] == 'custom' and 'expContent' in exp_result:
|
||||
with open(out_path, 'w') as f:
|
||||
f.write(exp_result['expContent'])
|
||||
return out_path
|
||||
elif exp_result['type'] == 'default' and 'expPath' in exp_result:
|
||||
# Optionally copy the file if outPath is different
|
||||
if exp_result['expPath'] != out_path:
|
||||
shutil.copyfile(exp_result['expPath'], out_path)
|
||||
return out_path
|
||||
else:
|
||||
raise Exception('Unknown expResult type or missing content')
|
||||
|
||||
def generate_yolox_inference_exp(training_id, options=None, use_base_config=False):
|
||||
"""Generate inference exp.py using DB values
|
||||
|
||||
Args:
|
||||
training_id: The training/project_details ID
|
||||
options: Optional overrides for data paths
|
||||
use_base_config: If True, load base config and only override with user-defined values
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
training = Training.query.get(training_id)
|
||||
if not training:
|
||||
training = Training.query.filter_by(project_details_id=training_id).first()
|
||||
|
||||
if not training:
|
||||
raise Exception(f'Training not found for trainingId or project_details_id: {training_id}')
|
||||
|
||||
# Always use the project_details_id for annotation file names and paths
|
||||
project_details_id = training.project_details_id
|
||||
|
||||
# Get annotation file names from options or use defaults
|
||||
# Use training.id (not project_details_id) for consistency with generate_training_json
|
||||
train_ann = options.get('train_ann', f'coco_project_{training_id}_train.json')
|
||||
val_ann = options.get('val_ann', f'coco_project_{training_id}_valid.json')
|
||||
test_ann = options.get('test_ann', f'coco_project_{training_id}_test.json')
|
||||
|
||||
# Get data_dir - this should point to where IMAGES are located (not annotations)
|
||||
# YOLOX will combine data_dir + file_name from COCO JSON to find images
|
||||
# The annotations are in a separate location (output folder)
|
||||
from services.settings_service import get_setting
|
||||
from models.TrainingProjectDetails import TrainingProjectDetails
|
||||
|
||||
if 'data_dir' in options:
|
||||
data_dir = options['data_dir']
|
||||
else:
|
||||
# Use the yolox_data_dir setting - this is where training images are stored
|
||||
data_dir = get_setting('yolox_data_dir', '/home/kitraining/To_Annotate/')
|
||||
# Ensure it ends with a separator
|
||||
if not data_dir.endswith(os.sep) and not data_dir.endswith('/'):
|
||||
data_dir += os.sep
|
||||
|
||||
# Get num_classes from ProjectClass table (3NF)
|
||||
num_classes = 80
|
||||
try:
|
||||
from models.ProjectClass import ProjectClass
|
||||
training_project = TrainingProject.query.get(project_details_id)
|
||||
if training_project:
|
||||
# Count classes from ProjectClass table
|
||||
class_count = ProjectClass.query.filter_by(project_id=training_project.project_id).count()
|
||||
if class_count > 0:
|
||||
num_classes = class_count
|
||||
except Exception as e:
|
||||
print(f'Could not determine num_classes from ProjectClass: {e}')
|
||||
|
||||
# Initialize config dictionary
|
||||
config = {}
|
||||
|
||||
# If using base config (transfer learning from COCO), load protected parameters first
|
||||
if use_base_config and training.selected_model:
|
||||
try:
|
||||
base_config = load_base_config(training.selected_model)
|
||||
config.update(base_config)
|
||||
print(f'Loaded base config for {training.selected_model}: {list(base_config.keys())}')
|
||||
except Exception as e:
|
||||
print(f'Warning: Could not load base config for {training.selected_model}: {e}')
|
||||
print('Falling back to custom settings only')
|
||||
|
||||
# Get size arrays from TrainingSize table (3NF)
|
||||
from models.TrainingSize import TrainingSize
|
||||
|
||||
def get_size_array(training_id, size_type):
|
||||
"""Helper to get size array from TrainingSize table"""
|
||||
sizes = TrainingSize.query.filter_by(
|
||||
training_id=training_id,
|
||||
size_type=size_type
|
||||
).order_by(TrainingSize.value_order).all()
|
||||
return [s.value for s in sizes] if sizes else None
|
||||
|
||||
input_size = get_size_array(training.id, 'input_size')
|
||||
test_size = get_size_array(training.id, 'test_size')
|
||||
mosaic_scale = get_size_array(training.id, 'mosaic_scale')
|
||||
mixup_scale = get_size_array(training.id, 'mixup_scale')
|
||||
|
||||
# Override with user-defined values from training table (only if they exist and are not None)
|
||||
user_overrides = {
|
||||
'depth': training.depth,
|
||||
'width': training.width,
|
||||
'input_size': input_size,
|
||||
'mosaic_scale': mosaic_scale,
|
||||
'test_size': test_size,
|
||||
'enable_mixup': training.enable_mixup,
|
||||
'max_epoch': training.max_epoch,
|
||||
'warmup_epochs': training.warmup_epochs,
|
||||
'warmup_lr': training.warmup_lr,
|
||||
'basic_lr_per_img': training.basic_lr_per_img,
|
||||
'scheduler': training.scheduler,
|
||||
'no_aug_epochs': training.no_aug_epochs,
|
||||
'min_lr_ratio': training.min_lr_ratio,
|
||||
'ema': training.ema,
|
||||
'weight_decay': training.weight_decay,
|
||||
'momentum': training.momentum,
|
||||
'print_interval': training.print_interval,
|
||||
'eval_interval': training.eval_interval,
|
||||
'test_conf': training.test_conf,
|
||||
'nms_thre': training.nms_thre,
|
||||
'mosaic_prob': training.mosaic_prob,
|
||||
'mixup_prob': training.mixup_prob,
|
||||
'hsv_prob': training.hsv_prob,
|
||||
'flip_prob': training.flip_prob,
|
||||
# Convert single values to tuples for YOLOX augmentation parameters
|
||||
'degrees': (training.degrees, training.degrees) if training.degrees is not None and not isinstance(training.degrees, (list, tuple)) else training.degrees,
|
||||
'translate': (training.translate, training.translate) if training.translate is not None and not isinstance(training.translate, (list, tuple)) else training.translate,
|
||||
'shear': (training.shear, training.shear) if training.shear is not None and not isinstance(training.shear, (list, tuple)) else training.shear,
|
||||
'mixup_scale': mixup_scale,
|
||||
'activation': training.activation,
|
||||
}
|
||||
|
||||
# Only override if value is explicitly set (not None)
|
||||
for key, value in user_overrides.items():
|
||||
if value is not None:
|
||||
config[key] = value
|
||||
|
||||
# Apply any additional options overrides
|
||||
config.update(options)
|
||||
|
||||
# Set defaults for any missing required parameters
|
||||
config.setdefault('depth', 1.00)
|
||||
config.setdefault('width', 1.00)
|
||||
config.setdefault('input_size', [640, 640])
|
||||
config.setdefault('mosaic_scale', [0.1, 2])
|
||||
config.setdefault('random_size', [10, 20])
|
||||
config.setdefault('test_size', [640, 640])
|
||||
config.setdefault('enable_mixup', False)
|
||||
config.setdefault('exp_name', 'inference_exp')
|
||||
|
||||
# Prepare data_dir for template - escape backslashes and remove trailing separator
|
||||
data_dir_clean = data_dir.rstrip('/\\')
|
||||
data_dir_escaped = data_dir_clean.replace('\\', '\\\\')
|
||||
|
||||
# Calculate annotations directory (where JSON files are stored)
|
||||
# This is in the output folder, not with the images
|
||||
from models.TrainingProjectDetails import TrainingProjectDetails
|
||||
details = TrainingProjectDetails.query.get(project_details_id)
|
||||
if details:
|
||||
training_project = TrainingProject.query.get(details.project_id)
|
||||
project_name = training_project.title.replace(' ', '_') if training_project and training_project.title else f'project_{details.project_id}'
|
||||
else:
|
||||
project_name = f'project_{project_details_id}'
|
||||
|
||||
training_folder_name = f"{training.exp_name or training.training_name or 'training'}_{training_id}"
|
||||
training_folder_name = training_folder_name.replace(' ', '_')
|
||||
|
||||
output_base_path = get_setting('yolox_output_path', './backend')
|
||||
annotations_parent_dir = os.path.join(output_base_path, project_name, training_folder_name)
|
||||
annotations_parent_escaped = annotations_parent_dir.replace('\\', '\\\\')
|
||||
|
||||
# Build exp content
|
||||
exp_content = f'''#!/usr/bin/env python3
|
||||
# -*- coding:utf-8 -*-
|
||||
# Copyright (c) Megvii, Inc. and its affiliates.
|
||||
|
||||
import os
|
||||
|
||||
from yolox.exp import Exp as MyExp
|
||||
|
||||
|
||||
class Exp(MyExp):
|
||||
def __init__(self):
|
||||
super(Exp, self).__init__()
|
||||
self.data_dir = "{data_dir_escaped}" # Where images are located
|
||||
self.annotations_dir = "{annotations_parent_escaped}" # Where annotation JSONs are located
|
||||
self.train_ann = "{train_ann}"
|
||||
self.val_ann = "{val_ann}"
|
||||
self.test_ann = "{test_ann}"
|
||||
self.num_classes = {num_classes}
|
||||
# Disable train2017 subdirectory - our images are directly in data_dir
|
||||
self.name = ""
|
||||
# Set data workers for training
|
||||
self.data_num_workers = 8
|
||||
'''
|
||||
|
||||
# Set pretrained_ckpt if transfer_learning is 'coco'
|
||||
if training.transfer_learning and isinstance(training.transfer_learning, str) and training.transfer_learning.lower() == 'coco':
|
||||
yolox_base_dir = '/home/kitraining/Yolox/YOLOX-main'
|
||||
selected_model = training.selected_model.replace('.pth', '') if training.selected_model else ''
|
||||
if selected_model:
|
||||
exp_content += f" self.pretrained_ckpt = r'{yolox_base_dir}/pretrained/{selected_model}.pth'\n"
|
||||
|
||||
# Format arrays
|
||||
def format_value(val):
|
||||
if isinstance(val, (list, tuple)):
|
||||
# Convert float values to int for size-related parameters
|
||||
formatted_items = []
|
||||
for item in val:
|
||||
# Convert to int if it's a whole number float
|
||||
if isinstance(item, float) and item.is_integer():
|
||||
formatted_items.append(str(int(item)))
|
||||
else:
|
||||
formatted_items.append(str(item))
|
||||
return '(' + ', '.join(formatted_items) + ')'
|
||||
elif isinstance(val, bool):
|
||||
return str(val)
|
||||
elif isinstance(val, str):
|
||||
return f'"{val}"'
|
||||
elif isinstance(val, float) and val.is_integer():
|
||||
# Convert whole number floats to ints
|
||||
return str(int(val))
|
||||
else:
|
||||
return str(val)
|
||||
|
||||
# Add all config parameters to exp
|
||||
for key, value in config.items():
|
||||
if key not in ['exp_name']: # exp_name is handled separately
|
||||
exp_content += f" self.{key} = {format_value(value)}\n"
|
||||
|
||||
# Add get_dataset override using name parameter for image directory
|
||||
exp_content += '''
|
||||
def get_dataset(self, cache=False, cache_type="ram"):
|
||||
"""Override to use name parameter for images directory"""
|
||||
from yolox.data import COCODataset
|
||||
|
||||
# COCODataset constructs image paths as: os.path.join(data_dir, name, file_name)
|
||||
# YOLOX adds "annotations/" to data_dir automatically, so we pass annotations_dir directly
|
||||
# Use empty string for name since we have absolute paths in JSON
|
||||
return COCODataset(
|
||||
data_dir=self.annotations_dir,
|
||||
json_file=self.train_ann,
|
||||
name="",
|
||||
img_size=self.input_size,
|
||||
preproc=self.preproc if hasattr(self, 'preproc') else None,
|
||||
cache=cache,
|
||||
cache_type=cache_type,
|
||||
)
|
||||
|
||||
def get_eval_dataset(self, **kwargs):
|
||||
"""Override eval dataset using name parameter"""
|
||||
from yolox.data import COCODataset
|
||||
|
||||
testdev = kwargs.get("testdev", False)
|
||||
legacy = kwargs.get("legacy", False)
|
||||
|
||||
return COCODataset(
|
||||
data_dir=self.annotations_dir,
|
||||
json_file=self.val_ann if not testdev else self.test_ann,
|
||||
name="",
|
||||
img_size=self.test_size,
|
||||
preproc=None, # No preprocessing for evaluation
|
||||
)
|
||||
'''
|
||||
|
||||
# Add exp_name at the end (uses dynamic path)
|
||||
exp_content += f''' self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]
|
||||
'''
|
||||
|
||||
return exp_content
|
||||
|
||||
def save_yolox_inference_exp(training_id, out_path, options=None):
|
||||
"""Save inference exp.py to custom path"""
|
||||
exp_content = generate_yolox_inference_exp(training_id, options, use_base_config=False)
|
||||
with open(out_path, 'w') as f:
|
||||
f.write(exp_content)
|
||||
return out_path
|
||||
|
||||
@@ -1,48 +1,48 @@
|
||||
const Training = require('../models/training.js');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
async function pushYoloxExpToDb(settings) {
|
||||
// Normalize boolean and array fields for DB
|
||||
const normalized = { ...settings };
|
||||
// Map 'act' from frontend to 'activation' for DB
|
||||
if (normalized.act !== undefined) {
|
||||
normalized.activation = normalized.act;
|
||||
delete normalized.act;
|
||||
}
|
||||
// Convert 'on'/'off' to boolean for save_history_ckpt
|
||||
if (typeof normalized.save_history_ckpt === 'string') {
|
||||
normalized.save_history_ckpt = normalized.save_history_ckpt === 'on' ? true : false;
|
||||
}
|
||||
// Convert comma-separated strings to arrays for input_size, test_size, mosaic_scale, mixup_scale
|
||||
['input_size', 'test_size', 'mosaic_scale', 'mixup_scale'].forEach(key => {
|
||||
if (typeof normalized[key] === 'string') {
|
||||
const arr = normalized[key].split(',').map(v => parseFloat(v.trim()));
|
||||
normalized[key] = arr.length === 1 ? arr[0] : arr;
|
||||
}
|
||||
});
|
||||
// Find TrainingProjectDetails for this project
|
||||
const TrainingProjectDetails = require('../models/TrainingProjectDetails.js');
|
||||
const details = await TrainingProjectDetails.findOne({ where: { project_id: normalized.project_id } });
|
||||
if (!details) throw new Error('TrainingProjectDetails not found for project_id ' + normalized.project_id);
|
||||
normalized.project_details_id = details.id;
|
||||
// Create DB row
|
||||
const training = await Training.create(normalized);
|
||||
return training;
|
||||
}
|
||||
|
||||
async function generateYoloxExpFromDb(trainingId) {
|
||||
// Fetch training row from DB
|
||||
const training = await Training.findByPk(trainingId);
|
||||
if (!training) throw new Error('Training not found');
|
||||
// Template for exp.py
|
||||
const expTemplate = `#!/usr/bin/env python3\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport os\nimport random\n\nimport torch\nimport torch.distributed as dist\nimport torch.nn as nn\n\nfrom .base_exp import BaseExp\n\n__all__ = [\"Exp\", \"check_exp_value\"]\n\nclass Exp(BaseExp):\n def __init__(self):\n super().__init__()\n\n # ---------------- model config ---------------- #\n self.num_classes = ${training.num_classes || 80}\n self.depth = ${training.depth || 1.00}\n self.width = ${training.width || 1.00}\n self.act = \"${training.activation || training.act || 'silu'}\"\n\n # ---------------- dataloader config ---------------- #\n self.data_num_workers = ${training.data_num_workers || 4}\n self.input_size = (${Array.isArray(training.input_size) ? training.input_size.join(', ') : '640, 640'})\n self.multiscale_range = ${training.multiscale_range || 5}\n self.data_dir = ${training.data_dir ? `\"${training.data_dir}\"` : 'None'}\n self.train_ann = \"${training.train_ann || 'instances_train2017.json'}\"\n self.val_ann = \"${training.val_ann || 'instances_val2017.json'}\"\n self.test_ann = \"${training.test_ann || 'instances_test2017.json'}\"\n\n # --------------- transform config ----------------- #\n self.mosaic_prob = ${training.mosaic_prob !== undefined ? training.mosaic_prob : 1.0}\n self.mixup_prob = ${training.mixup_prob !== undefined ? training.mixup_prob : 1.0}\n self.hsv_prob = ${training.hsv_prob !== undefined ? training.hsv_prob : 1.0}\n self.flip_prob = ${training.flip_prob !== undefined ? training.flip_prob : 0.5}\n self.degrees = ${training.degrees !== undefined ? training.degrees : 10.0}\n self.translate = ${training.translate !== undefined ? training.translate : 0.1}\n self.mosaic_scale = (${Array.isArray(training.mosaic_scale) ? training.mosaic_scale.join(', ') : '0.1, 2'})\n self.enable_mixup = ${training.enable_mixup !== undefined ? training.enable_mixup : true}\n self.mixup_scale = (${Array.isArray(training.mixup_scale) ? training.mixup_scale.join(', ') : '0.5, 1.5'})\n self.shear = ${training.shear !== undefined ? training.shear : 2.0}\n\n # -------------- training config --------------------- #\n self.warmup_epochs = ${training.warmup_epochs !== undefined ? training.warmup_epochs : 5}\n self.max_epoch = ${training.max_epoch !== undefined ? training.max_epoch : 300}\n self.warmup_lr = ${training.warmup_lr !== undefined ? training.warmup_lr : 0}\n self.min_lr_ratio = ${training.min_lr_ratio !== undefined ? training.min_lr_ratio : 0.05}\n self.basic_lr_per_img = ${training.basic_lr_per_img !== undefined ? training.basic_lr_per_img : 0.01 / 64.0}\n self.scheduler = \"${training.scheduler || 'yoloxwarmcos'}\"\n self.no_aug_epochs = ${training.no_aug_epochs !== undefined ? training.no_aug_epochs : 15}\n self.ema = ${training.ema !== undefined ? training.ema : true}\n self.weight_decay = ${training.weight_decay !== undefined ? training.weight_decay : 5e-4}\n self.momentum = ${training.momentum !== undefined ? training.momentum : 0.9}\n self.print_interval = ${training.print_interval !== undefined ? training.print_interval : 10}\n self.eval_interval = ${training.eval_interval !== undefined ? training.eval_interval : 10}\n self.save_history_ckpt = ${training.save_history_ckpt !== undefined ? training.save_history_ckpt : true}\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n\n # ----------------- testing config ------------------ #\n self.test_size = (${Array.isArray(training.test_size) ? training.test_size.join(', ') : '640, 640'})\n self.test_conf = ${training.test_conf !== undefined ? training.test_conf : 0.01}\n self.nmsthre = ${training.nmsthre !== undefined ? training.nmsthre : 0.65}\n\n # ... rest of the template ...\n\ndef check_exp_value(exp: Exp):\n h, w = exp.input_size\n assert h % 32 == 0 and w % 32 == 0, \"input size must be multiples of 32\"\n`;
|
||||
// Save to file in output directory
|
||||
const outDir = path.join(__dirname, '../../', training.project_id ? `project_${training.project_id}/${trainingId}` : 'exp_files');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
const filePath = path.join(outDir, 'exp.py');
|
||||
fs.writeFileSync(filePath, expTemplate);
|
||||
return filePath;
|
||||
}
|
||||
|
||||
const Training = require('../models/training.js');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
async function pushYoloxExpToDb(settings) {
|
||||
// Normalize boolean and array fields for DB
|
||||
const normalized = { ...settings };
|
||||
// Map 'act' from frontend to 'activation' for DB
|
||||
if (normalized.act !== undefined) {
|
||||
normalized.activation = normalized.act;
|
||||
delete normalized.act;
|
||||
}
|
||||
// Convert 'on'/'off' to boolean for save_history_ckpt
|
||||
if (typeof normalized.save_history_ckpt === 'string') {
|
||||
normalized.save_history_ckpt = normalized.save_history_ckpt === 'on' ? true : false;
|
||||
}
|
||||
// Convert comma-separated strings to arrays for input_size, test_size, mosaic_scale, mixup_scale
|
||||
['input_size', 'test_size', 'mosaic_scale', 'mixup_scale'].forEach(key => {
|
||||
if (typeof normalized[key] === 'string') {
|
||||
const arr = normalized[key].split(',').map(v => parseFloat(v.trim()));
|
||||
normalized[key] = arr.length === 1 ? arr[0] : arr;
|
||||
}
|
||||
});
|
||||
// Find TrainingProjectDetails for this project
|
||||
const TrainingProjectDetails = require('../models/TrainingProjectDetails.js');
|
||||
const details = await TrainingProjectDetails.findOne({ where: { project_id: normalized.project_id } });
|
||||
if (!details) throw new Error('TrainingProjectDetails not found for project_id ' + normalized.project_id);
|
||||
normalized.project_details_id = details.id;
|
||||
// Create DB row
|
||||
const training = await Training.create(normalized);
|
||||
return training;
|
||||
}
|
||||
|
||||
async function generateYoloxExpFromDb(trainingId) {
|
||||
// Fetch training row from DB
|
||||
const training = await Training.findByPk(trainingId);
|
||||
if (!training) throw new Error('Training not found');
|
||||
// Template for exp.py
|
||||
const expTemplate = `#!/usr/bin/env python3\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport os\nimport random\n\nimport torch\nimport torch.distributed as dist\nimport torch.nn as nn\n\nfrom .base_exp import BaseExp\n\n__all__ = [\"Exp\", \"check_exp_value\"]\n\nclass Exp(BaseExp):\n def __init__(self):\n super().__init__()\n\n # ---------------- model config ---------------- #\n self.num_classes = ${training.num_classes || 80}\n self.depth = ${training.depth || 1.00}\n self.width = ${training.width || 1.00}\n self.act = \"${training.activation || training.act || 'silu'}\"\n\n # ---------------- dataloader config ---------------- #\n self.data_num_workers = ${training.data_num_workers || 4}\n self.input_size = (${Array.isArray(training.input_size) ? training.input_size.join(', ') : '640, 640'})\n self.multiscale_range = ${training.multiscale_range || 5}\n self.data_dir = ${training.data_dir ? `\"${training.data_dir}\"` : 'None'}\n self.train_ann = \"${training.train_ann || 'instances_train2017.json'}\"\n self.val_ann = \"${training.val_ann || 'instances_val2017.json'}\"\n self.test_ann = \"${training.test_ann || 'instances_test2017.json'}\"\n\n # --------------- transform config ----------------- #\n self.mosaic_prob = ${training.mosaic_prob !== undefined ? training.mosaic_prob : 1.0}\n self.mixup_prob = ${training.mixup_prob !== undefined ? training.mixup_prob : 1.0}\n self.hsv_prob = ${training.hsv_prob !== undefined ? training.hsv_prob : 1.0}\n self.flip_prob = ${training.flip_prob !== undefined ? training.flip_prob : 0.5}\n self.degrees = ${training.degrees !== undefined ? training.degrees : 10.0}\n self.translate = ${training.translate !== undefined ? training.translate : 0.1}\n self.mosaic_scale = (${Array.isArray(training.mosaic_scale) ? training.mosaic_scale.join(', ') : '0.1, 2'})\n self.enable_mixup = ${training.enable_mixup !== undefined ? training.enable_mixup : true}\n self.mixup_scale = (${Array.isArray(training.mixup_scale) ? training.mixup_scale.join(', ') : '0.5, 1.5'})\n self.shear = ${training.shear !== undefined ? training.shear : 2.0}\n\n # -------------- training config --------------------- #\n self.warmup_epochs = ${training.warmup_epochs !== undefined ? training.warmup_epochs : 5}\n self.max_epoch = ${training.max_epoch !== undefined ? training.max_epoch : 300}\n self.warmup_lr = ${training.warmup_lr !== undefined ? training.warmup_lr : 0}\n self.min_lr_ratio = ${training.min_lr_ratio !== undefined ? training.min_lr_ratio : 0.05}\n self.basic_lr_per_img = ${training.basic_lr_per_img !== undefined ? training.basic_lr_per_img : 0.01 / 64.0}\n self.scheduler = \"${training.scheduler || 'yoloxwarmcos'}\"\n self.no_aug_epochs = ${training.no_aug_epochs !== undefined ? training.no_aug_epochs : 15}\n self.ema = ${training.ema !== undefined ? training.ema : true}\n self.weight_decay = ${training.weight_decay !== undefined ? training.weight_decay : 5e-4}\n self.momentum = ${training.momentum !== undefined ? training.momentum : 0.9}\n self.print_interval = ${training.print_interval !== undefined ? training.print_interval : 10}\n self.eval_interval = ${training.eval_interval !== undefined ? training.eval_interval : 10}\n self.save_history_ckpt = ${training.save_history_ckpt !== undefined ? training.save_history_ckpt : true}\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n\n # ----------------- testing config ------------------ #\n self.test_size = (${Array.isArray(training.test_size) ? training.test_size.join(', ') : '640, 640'})\n self.test_conf = ${training.test_conf !== undefined ? training.test_conf : 0.01}\n self.nmsthre = ${training.nmsthre !== undefined ? training.nmsthre : 0.65}\n\n # ... rest of the template ...\n\ndef check_exp_value(exp: Exp):\n h, w = exp.input_size\n assert h % 32 == 0 and w % 32 == 0, \"input size must be multiples of 32\"\n`;
|
||||
// Save to file in output directory
|
||||
const outDir = path.join(__dirname, '../../', training.project_id ? `project_${training.project_id}/${trainingId}` : 'exp_files');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
const filePath = path.join(outDir, 'exp.py');
|
||||
fs.writeFileSync(filePath, expTemplate);
|
||||
return filePath;
|
||||
}
|
||||
|
||||
module.exports = { pushYoloxExpToDb, generateYoloxExpFromDb };
|
||||
@@ -1,92 +1,113 @@
|
||||
from models.training import Training
|
||||
from models.TrainingProjectDetails import TrainingProjectDetails
|
||||
from database.database import db
|
||||
|
||||
def push_yolox_exp_to_db(settings):
|
||||
"""Save YOLOX settings to database"""
|
||||
normalized = dict(settings)
|
||||
|
||||
# Map common frontend aliases to DB column names
|
||||
alias_map = {
|
||||
'act': 'activation',
|
||||
'nmsthre': 'nms_thre',
|
||||
'select_model': 'selected_model'
|
||||
}
|
||||
for a, b in alias_map.items():
|
||||
if a in normalized and b not in normalized:
|
||||
normalized[b] = normalized.pop(a)
|
||||
|
||||
# Convert 'on'/'off' or 'true'/'false' strings to boolean for known boolean fields
|
||||
for bool_field in ['save_history_ckpt', 'ema', 'enable_mixup']:
|
||||
if bool_field in normalized:
|
||||
val = normalized[bool_field]
|
||||
if isinstance(val, str):
|
||||
normalized[bool_field] = val.lower() in ('1', 'true', 'on')
|
||||
else:
|
||||
normalized[bool_field] = bool(val)
|
||||
|
||||
# Convert comma-separated strings to arrays for JSON fields
|
||||
for key in ['input_size', 'test_size', 'mosaic_scale', 'mixup_scale']:
|
||||
if key in normalized and isinstance(normalized[key], str):
|
||||
parts = [p.strip() for p in normalized[key].split(',') if p.strip()]
|
||||
try:
|
||||
arr = [float(p) for p in parts]
|
||||
except Exception:
|
||||
arr = parts
|
||||
normalized[key] = arr[0] if len(arr) == 1 else arr
|
||||
|
||||
# Ensure we have a TrainingProjectDetails row for project_id
|
||||
project_id = normalized.get('project_id')
|
||||
if not project_id:
|
||||
raise Exception('Missing project_id in settings')
|
||||
details = TrainingProjectDetails.query.filter_by(project_id=project_id).first()
|
||||
if not details:
|
||||
raise Exception(f'TrainingProjectDetails not found for project_id {project_id}')
|
||||
normalized['project_details_id'] = details.id
|
||||
|
||||
# Filter normalized to only columns that exist on the Training model
|
||||
valid_cols = {c.name: c for c in Training.__table__.columns}
|
||||
filtered = {}
|
||||
for k, v in normalized.items():
|
||||
if k in valid_cols:
|
||||
col_type = valid_cols[k].type.__class__.__name__
|
||||
# Try to coerce types for numeric/boolean columns
|
||||
try:
|
||||
if 'Integer' in col_type:
|
||||
if v is None or v == '':
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = int(float(v))
|
||||
elif 'Float' in col_type:
|
||||
if v is None or v == '':
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = float(v)
|
||||
elif 'Boolean' in col_type:
|
||||
if isinstance(v, str):
|
||||
filtered[k] = v.lower() in ('1', 'true', 'on')
|
||||
else:
|
||||
filtered[k] = bool(v)
|
||||
elif 'JSON' in col_type:
|
||||
filtered[k] = v
|
||||
elif 'LargeBinary' in col_type:
|
||||
# If a file path was passed, store its bytes; otherwise store raw bytes
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
filtered[k] = v.encode('utf-8')
|
||||
except Exception:
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = v
|
||||
else:
|
||||
filtered[k] = v
|
||||
except Exception:
|
||||
# If conversion fails, just assign raw value
|
||||
filtered[k] = v
|
||||
|
||||
# Create DB row
|
||||
training = Training(**filtered)
|
||||
db.session.add(training)
|
||||
db.session.commit()
|
||||
|
||||
return training
|
||||
from models.training import Training
|
||||
from models.TrainingProjectDetails import TrainingProjectDetails
|
||||
from models.TrainingSize import TrainingSize
|
||||
from database.database import db
|
||||
|
||||
def push_yolox_exp_to_db(settings):
|
||||
"""Save YOLOX settings to database"""
|
||||
normalized = dict(settings)
|
||||
|
||||
# Map common frontend aliases to DB column names
|
||||
alias_map = {
|
||||
'act': 'activation',
|
||||
'nmsthre': 'nms_thre',
|
||||
'select_model': 'selected_model'
|
||||
}
|
||||
for a, b in alias_map.items():
|
||||
if a in normalized and b not in normalized:
|
||||
normalized[b] = normalized.pop(a)
|
||||
|
||||
# Convert 'on'/'off' or 'true'/'false' strings to boolean for known boolean fields
|
||||
for bool_field in ['save_history_ckpt', 'ema', 'enable_mixup']:
|
||||
if bool_field in normalized:
|
||||
val = normalized[bool_field]
|
||||
if isinstance(val, str):
|
||||
normalized[bool_field] = val.lower() in ('1', 'true', 'on')
|
||||
else:
|
||||
normalized[bool_field] = bool(val)
|
||||
|
||||
# Extract size arrays for separate TrainingSize table (3NF)
|
||||
size_arrays = {}
|
||||
for key in ['input_size', 'test_size', 'mosaic_scale', 'mixup_scale']:
|
||||
if key in normalized:
|
||||
if isinstance(normalized[key], str):
|
||||
parts = [p.strip() for p in normalized[key].split(',') if p.strip()]
|
||||
try:
|
||||
arr = [float(p) for p in parts]
|
||||
except Exception:
|
||||
arr = parts
|
||||
size_arrays[key] = arr[0] if len(arr) == 1 else (arr if isinstance(arr, list) else [arr])
|
||||
elif isinstance(normalized[key], list):
|
||||
size_arrays[key] = normalized[key]
|
||||
elif normalized[key] is not None:
|
||||
size_arrays[key] = [float(normalized[key])]
|
||||
# Remove from normalized dict since it won't be stored in training table
|
||||
del normalized[key]
|
||||
|
||||
# Ensure we have a TrainingProjectDetails row for project_id
|
||||
project_id = normalized.get('project_id')
|
||||
if not project_id:
|
||||
raise Exception('Missing project_id in settings')
|
||||
details = TrainingProjectDetails.query.filter_by(project_id=project_id).first()
|
||||
if not details:
|
||||
raise Exception(f'TrainingProjectDetails not found for project_id {project_id}')
|
||||
normalized['project_details_id'] = details.id
|
||||
|
||||
# Filter normalized to only columns that exist on the Training model
|
||||
valid_cols = {c.name: c for c in Training.__table__.columns}
|
||||
filtered = {}
|
||||
for k, v in normalized.items():
|
||||
if k in valid_cols:
|
||||
col_type = valid_cols[k].type.__class__.__name__
|
||||
# Try to coerce types for numeric/boolean columns
|
||||
try:
|
||||
if 'Integer' in col_type:
|
||||
if v is None or v == '':
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = int(float(v))
|
||||
elif 'Float' in col_type:
|
||||
if v is None or v == '':
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = float(v)
|
||||
elif 'Boolean' in col_type:
|
||||
if isinstance(v, str):
|
||||
filtered[k] = v.lower() in ('1', 'true', 'on')
|
||||
else:
|
||||
filtered[k] = bool(v)
|
||||
elif 'LargeBinary' in col_type:
|
||||
# If a file path was passed, store its bytes; otherwise store raw bytes
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
filtered[k] = v.encode('utf-8')
|
||||
except Exception:
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = v
|
||||
else:
|
||||
filtered[k] = v
|
||||
except Exception:
|
||||
# If conversion fails, just assign raw value
|
||||
filtered[k] = v
|
||||
|
||||
# Create DB row
|
||||
training = Training(**filtered)
|
||||
db.session.add(training)
|
||||
db.session.flush() # Get training.id
|
||||
|
||||
# Save size arrays to TrainingSize table (3NF)
|
||||
for size_type, values in size_arrays.items():
|
||||
if values and isinstance(values, list):
|
||||
for order, value in enumerate(values):
|
||||
size_record = TrainingSize(
|
||||
training_id=training.id,
|
||||
size_type=size_type,
|
||||
value_order=order,
|
||||
value=float(value)
|
||||
)
|
||||
db.session.add(size_record)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return training
|
||||
|
||||
@@ -1,120 +1,120 @@
|
||||
const sequelize = require('../database/database.js');
|
||||
const { Project, Img, Ann } = require('../models');
|
||||
const { fetchLableStudioProject, fetchProjectIdsAndTitles } = require('./fetch-labelstudio.js');
|
||||
|
||||
const updateStatus = { running: false };
|
||||
|
||||
async function seedLabelStudio() {
|
||||
updateStatus.running = true;
|
||||
console.log('Seeding started');
|
||||
try {
|
||||
await sequelize.sync();
|
||||
const projects = await fetchProjectIdsAndTitles();
|
||||
|
||||
for (const project of projects) {
|
||||
console.log(`Processing project ${project.id} (${project.title})`);
|
||||
|
||||
// Upsert project in DB
|
||||
await Project.upsert({ project_id: project.id, title: project.title });
|
||||
|
||||
// Fetch project data (annotations array)
|
||||
const data = await fetchLableStudioProject(project.id);
|
||||
if (!Array.isArray(data) || data.length === 0) {
|
||||
console.log(`No annotation data for project ${project.id}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Remove old images and annotations for this project
|
||||
const oldImages = await Img.findAll({ where: { project_id: project.id } });
|
||||
const oldImageIds = oldImages.map(img => img.image_id);
|
||||
if (oldImageIds.length > 0) {
|
||||
await Ann.destroy({ where: { image_id: oldImageIds } });
|
||||
await Img.destroy({ where: { project_id: project.id } });
|
||||
console.log(`Deleted ${oldImageIds.length} old images and their annotations for project ${project.id}`);
|
||||
}
|
||||
|
||||
// Prepare arrays
|
||||
const imagesBulk = [];
|
||||
const annsBulk = [];
|
||||
|
||||
for (const ann of data) {
|
||||
// Extract width/height
|
||||
let width = null;
|
||||
let height = null;
|
||||
if (Array.isArray(ann.label_rectangles) && ann.label_rectangles.length > 0) {
|
||||
width = ann.label_rectangles[0].original_width;
|
||||
height = ann.label_rectangles[0].original_height;
|
||||
} else if (Array.isArray(ann.label) && ann.label.length > 0 && ann.label[0].original_width && ann.label[0].original_height) {
|
||||
width = ann.label[0].original_width;
|
||||
height = ann.label[0].original_height;
|
||||
}
|
||||
|
||||
// Only push image and annotations if width and height are valid
|
||||
if (width && height) {
|
||||
imagesBulk.push({
|
||||
project_id: project.id,
|
||||
image_path: ann.image,
|
||||
width,
|
||||
height
|
||||
});
|
||||
|
||||
// Handle multiple annotations per image
|
||||
if (Array.isArray(ann.label_rectangles)) {
|
||||
for (const ann_detail of ann.label_rectangles) {
|
||||
annsBulk.push({
|
||||
image_path: ann.image,
|
||||
x: (ann_detail.x * width) / 100,
|
||||
y: (ann_detail.y * height) / 100,
|
||||
width: (ann_detail.width * width) / 100,
|
||||
height: (ann_detail.height * height) / 100,
|
||||
Label: Array.isArray(ann_detail.rectanglelabels) ? (ann_detail.rectanglelabels[0] || 'unknown') : (ann_detail.rectanglelabels || 'unknown')
|
||||
});
|
||||
}
|
||||
} else if (Array.isArray(ann.label)) {
|
||||
for (const ann_detail of ann.label) {
|
||||
annsBulk.push({
|
||||
image_path: ann.image,
|
||||
x: (ann_detail.x * width) / 100,
|
||||
y: (ann_detail.y * height) / 100,
|
||||
width: (ann_detail.width * width) / 100,
|
||||
height: (ann_detail.height * height) / 100,
|
||||
Label: Array.isArray(ann_detail.rectanglelabels) ? (ann_detail.rectanglelabels[0] || 'unknown') : (ann_detail.rectanglelabels || 'unknown')
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1) Insert images and get generated IDs
|
||||
const insertedImages = await Img.bulkCreate(imagesBulk, { returning: true });
|
||||
|
||||
// 2) Map image_path -> image_id
|
||||
const imageMap = {};
|
||||
for (const img of insertedImages) {
|
||||
imageMap[img.image_path] = img.image_id;
|
||||
}
|
||||
|
||||
// 3) Assign correct image_id to each annotation
|
||||
for (const ann of annsBulk) {
|
||||
ann.image_id = imageMap[ann.image_path];
|
||||
delete ann.image_path; // cleanup
|
||||
}
|
||||
|
||||
// 4) Insert annotations
|
||||
await Ann.bulkCreate(annsBulk);
|
||||
|
||||
console.log(`Inserted ${imagesBulk.length} images and ${annsBulk.length} annotations for project ${project.id}`);
|
||||
}
|
||||
|
||||
console.log('Seeding done');
|
||||
return { success: true, message: 'Data inserted successfully!' };
|
||||
} catch (error) {
|
||||
console.error('Error inserting data:', error);
|
||||
return { success: false, message: error.message };
|
||||
} finally {
|
||||
updateStatus.running = false;
|
||||
console.log('updateStatus.running set to false');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { seedLabelStudio, updateStatus };
|
||||
const sequelize = require('../database/database.js');
|
||||
const { Project, Img, Ann } = require('../models');
|
||||
const { fetchLableStudioProject, fetchProjectIdsAndTitles } = require('./fetch-labelstudio.js');
|
||||
|
||||
const updateStatus = { running: false };
|
||||
|
||||
async function seedLabelStudio() {
|
||||
updateStatus.running = true;
|
||||
console.log('Seeding started');
|
||||
try {
|
||||
await sequelize.sync();
|
||||
const projects = await fetchProjectIdsAndTitles();
|
||||
|
||||
for (const project of projects) {
|
||||
console.log(`Processing project ${project.id} (${project.title})`);
|
||||
|
||||
// Upsert project in DB
|
||||
await Project.upsert({ project_id: project.id, title: project.title });
|
||||
|
||||
// Fetch project data (annotations array)
|
||||
const data = await fetchLableStudioProject(project.id);
|
||||
if (!Array.isArray(data) || data.length === 0) {
|
||||
console.log(`No annotation data for project ${project.id}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Remove old images and annotations for this project
|
||||
const oldImages = await Img.findAll({ where: { project_id: project.id } });
|
||||
const oldImageIds = oldImages.map(img => img.image_id);
|
||||
if (oldImageIds.length > 0) {
|
||||
await Ann.destroy({ where: { image_id: oldImageIds } });
|
||||
await Img.destroy({ where: { project_id: project.id } });
|
||||
console.log(`Deleted ${oldImageIds.length} old images and their annotations for project ${project.id}`);
|
||||
}
|
||||
|
||||
// Prepare arrays
|
||||
const imagesBulk = [];
|
||||
const annsBulk = [];
|
||||
|
||||
for (const ann of data) {
|
||||
// Extract width/height
|
||||
let width = null;
|
||||
let height = null;
|
||||
if (Array.isArray(ann.label_rectangles) && ann.label_rectangles.length > 0) {
|
||||
width = ann.label_rectangles[0].original_width;
|
||||
height = ann.label_rectangles[0].original_height;
|
||||
} else if (Array.isArray(ann.label) && ann.label.length > 0 && ann.label[0].original_width && ann.label[0].original_height) {
|
||||
width = ann.label[0].original_width;
|
||||
height = ann.label[0].original_height;
|
||||
}
|
||||
|
||||
// Only push image and annotations if width and height are valid
|
||||
if (width && height) {
|
||||
imagesBulk.push({
|
||||
project_id: project.id,
|
||||
image_path: ann.image,
|
||||
width,
|
||||
height
|
||||
});
|
||||
|
||||
// Handle multiple annotations per image
|
||||
if (Array.isArray(ann.label_rectangles)) {
|
||||
for (const ann_detail of ann.label_rectangles) {
|
||||
annsBulk.push({
|
||||
image_path: ann.image,
|
||||
x: (ann_detail.x * width) / 100,
|
||||
y: (ann_detail.y * height) / 100,
|
||||
width: (ann_detail.width * width) / 100,
|
||||
height: (ann_detail.height * height) / 100,
|
||||
Label: Array.isArray(ann_detail.rectanglelabels) ? (ann_detail.rectanglelabels[0] || 'unknown') : (ann_detail.rectanglelabels || 'unknown')
|
||||
});
|
||||
}
|
||||
} else if (Array.isArray(ann.label)) {
|
||||
for (const ann_detail of ann.label) {
|
||||
annsBulk.push({
|
||||
image_path: ann.image,
|
||||
x: (ann_detail.x * width) / 100,
|
||||
y: (ann_detail.y * height) / 100,
|
||||
width: (ann_detail.width * width) / 100,
|
||||
height: (ann_detail.height * height) / 100,
|
||||
Label: Array.isArray(ann_detail.rectanglelabels) ? (ann_detail.rectanglelabels[0] || 'unknown') : (ann_detail.rectanglelabels || 'unknown')
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1) Insert images and get generated IDs
|
||||
const insertedImages = await Img.bulkCreate(imagesBulk, { returning: true });
|
||||
|
||||
// 2) Map image_path -> image_id
|
||||
const imageMap = {};
|
||||
for (const img of insertedImages) {
|
||||
imageMap[img.image_path] = img.image_id;
|
||||
}
|
||||
|
||||
// 3) Assign correct image_id to each annotation
|
||||
for (const ann of annsBulk) {
|
||||
ann.image_id = imageMap[ann.image_path];
|
||||
delete ann.image_path; // cleanup
|
||||
}
|
||||
|
||||
// 4) Insert annotations
|
||||
await Ann.bulkCreate(annsBulk);
|
||||
|
||||
console.log(`Inserted ${imagesBulk.length} images and ${annsBulk.length} annotations for project ${project.id}`);
|
||||
}
|
||||
|
||||
console.log('Seeding done');
|
||||
return { success: true, message: 'Data inserted successfully!' };
|
||||
} catch (error) {
|
||||
console.error('Error inserting data:', error);
|
||||
return { success: false, message: error.message };
|
||||
} finally {
|
||||
updateStatus.running = false;
|
||||
console.log('updateStatus.running set to false');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { seedLabelStudio, updateStatus };
|
||||
|
||||
@@ -1,149 +1,149 @@
|
||||
from database.database import db
|
||||
from models.LabelStudioProject import LabelStudioProject
|
||||
from models.Images import Image
|
||||
from models.Annotation import Annotation
|
||||
from services.fetch_labelstudio import fetch_label_studio_project, fetch_project_ids_and_titles
|
||||
|
||||
update_status = {"running": False}
|
||||
|
||||
def seed_label_studio():
|
||||
"""Seed database with Label Studio project data"""
|
||||
update_status["running"] = True
|
||||
print('Seeding started')
|
||||
|
||||
try:
|
||||
projects = fetch_project_ids_and_titles()
|
||||
|
||||
for project in projects:
|
||||
print(f"Processing project {project['id']} ({project['title']})")
|
||||
|
||||
# Upsert project in DB
|
||||
existing_project = LabelStudioProject.query.filter_by(project_id=project['id']).first()
|
||||
if existing_project:
|
||||
existing_project.title = project['title']
|
||||
else:
|
||||
new_project = LabelStudioProject(project_id=project['id'], title=project['title'])
|
||||
db.session.add(new_project)
|
||||
db.session.commit()
|
||||
|
||||
# Fetch project data (annotations array)
|
||||
data = fetch_label_studio_project(project['id'])
|
||||
if not isinstance(data, list) or len(data) == 0:
|
||||
print(f"No annotation data for project {project['id']}")
|
||||
continue
|
||||
|
||||
# Remove old images and annotations for this project
|
||||
old_images = Image.query.filter_by(project_id=project['id']).all()
|
||||
old_image_ids = [img.image_id for img in old_images]
|
||||
|
||||
if old_image_ids:
|
||||
Annotation.query.filter(Annotation.image_id.in_(old_image_ids)).delete(synchronize_session=False)
|
||||
Image.query.filter_by(project_id=project['id']).delete()
|
||||
db.session.commit()
|
||||
print(f"Deleted {len(old_image_ids)} old images and their annotations for project {project['id']}")
|
||||
|
||||
# Prepare arrays
|
||||
images_bulk = []
|
||||
anns_bulk = []
|
||||
|
||||
for ann in data:
|
||||
# Extract width/height
|
||||
width = None
|
||||
height = None
|
||||
|
||||
if isinstance(ann.get('label_rectangles'), list) and len(ann['label_rectangles']) > 0:
|
||||
width = ann['label_rectangles'][0].get('original_width')
|
||||
height = ann['label_rectangles'][0].get('original_height')
|
||||
elif isinstance(ann.get('label'), list) and len(ann['label']) > 0:
|
||||
if ann['label'][0].get('original_width') and ann['label'][0].get('original_height'):
|
||||
width = ann['label'][0]['original_width']
|
||||
height = ann['label'][0]['original_height']
|
||||
|
||||
# Only process if width and height are valid
|
||||
if width and height:
|
||||
image_data = {
|
||||
'project_id': project['id'],
|
||||
'image_path': ann.get('image'),
|
||||
'width': width,
|
||||
'height': height
|
||||
}
|
||||
images_bulk.append(image_data)
|
||||
|
||||
# Handle multiple annotations per image
|
||||
if isinstance(ann.get('label_rectangles'), list):
|
||||
for ann_detail in ann['label_rectangles']:
|
||||
# Get label safely
|
||||
rectanglelabels = ann_detail.get('rectanglelabels', [])
|
||||
if isinstance(rectanglelabels, list) and len(rectanglelabels) > 0:
|
||||
label = rectanglelabels[0]
|
||||
elif isinstance(rectanglelabels, str):
|
||||
label = rectanglelabels
|
||||
else:
|
||||
label = 'unknown'
|
||||
|
||||
ann_data = {
|
||||
'image_path': ann.get('image'),
|
||||
'x': (ann_detail['x'] * width) / 100,
|
||||
'y': (ann_detail['y'] * height) / 100,
|
||||
'width': (ann_detail['width'] * width) / 100,
|
||||
'height': (ann_detail['height'] * height) / 100,
|
||||
'Label': label
|
||||
}
|
||||
anns_bulk.append(ann_data)
|
||||
elif isinstance(ann.get('label'), list):
|
||||
for ann_detail in ann['label']:
|
||||
# Get label safely
|
||||
rectanglelabels = ann_detail.get('rectanglelabels', [])
|
||||
if isinstance(rectanglelabels, list) and len(rectanglelabels) > 0:
|
||||
label = rectanglelabels[0]
|
||||
elif isinstance(rectanglelabels, str):
|
||||
label = rectanglelabels
|
||||
else:
|
||||
label = 'unknown'
|
||||
|
||||
ann_data = {
|
||||
'image_path': ann.get('image'),
|
||||
'x': (ann_detail['x'] * width) / 100,
|
||||
'y': (ann_detail['y'] * height) / 100,
|
||||
'width': (ann_detail['width'] * width) / 100,
|
||||
'height': (ann_detail['height'] * height) / 100,
|
||||
'Label': label
|
||||
}
|
||||
anns_bulk.append(ann_data)
|
||||
|
||||
# Insert images and get generated IDs
|
||||
inserted_images = []
|
||||
for img_data in images_bulk:
|
||||
new_image = Image(**img_data)
|
||||
db.session.add(new_image)
|
||||
db.session.flush() # Flush to get the ID
|
||||
inserted_images.append(new_image)
|
||||
db.session.commit()
|
||||
|
||||
# Map image_path -> image_id
|
||||
image_map = {img.image_path: img.image_id for img in inserted_images}
|
||||
|
||||
# Assign correct image_id to each annotation
|
||||
for ann_data in anns_bulk:
|
||||
ann_data['image_id'] = image_map.get(ann_data['image_path'])
|
||||
del ann_data['image_path']
|
||||
|
||||
# Insert annotations
|
||||
for ann_data in anns_bulk:
|
||||
new_annotation = Annotation(**ann_data)
|
||||
db.session.add(new_annotation)
|
||||
db.session.commit()
|
||||
|
||||
print(f"Inserted {len(images_bulk)} images and {len(anns_bulk)} annotations for project {project['id']}")
|
||||
|
||||
print('Seeding done')
|
||||
return {'success': True, 'message': 'Data inserted successfully!'}
|
||||
|
||||
except Exception as error:
|
||||
print(f'Error inserting data: {error}')
|
||||
db.session.rollback()
|
||||
return {'success': False, 'message': str(error)}
|
||||
|
||||
finally:
|
||||
update_status["running"] = False
|
||||
print('updateStatus.running set to false')
|
||||
from database.database import db
|
||||
from models.LabelStudioProject import LabelStudioProject
|
||||
from models.Images import Image
|
||||
from models.Annotation import Annotation
|
||||
from services.fetch_labelstudio import fetch_label_studio_project, fetch_project_ids_and_titles
|
||||
|
||||
update_status = {"running": False}
|
||||
|
||||
def seed_label_studio():
|
||||
"""Seed database with Label Studio project data"""
|
||||
update_status["running"] = True
|
||||
print('Seeding started')
|
||||
|
||||
try:
|
||||
projects = fetch_project_ids_and_titles()
|
||||
|
||||
for project in projects:
|
||||
print(f"Processing project {project['id']} ({project['title']})")
|
||||
|
||||
# Upsert project in DB
|
||||
existing_project = LabelStudioProject.query.filter_by(project_id=project['id']).first()
|
||||
if existing_project:
|
||||
existing_project.title = project['title']
|
||||
else:
|
||||
new_project = LabelStudioProject(project_id=project['id'], title=project['title'])
|
||||
db.session.add(new_project)
|
||||
db.session.commit()
|
||||
|
||||
# Fetch project data (annotations array)
|
||||
data = fetch_label_studio_project(project['id'])
|
||||
if not isinstance(data, list) or len(data) == 0:
|
||||
print(f"No annotation data for project {project['id']}")
|
||||
continue
|
||||
|
||||
# Remove old images and annotations for this project
|
||||
old_images = Image.query.filter_by(project_id=project['id']).all()
|
||||
old_image_ids = [img.image_id for img in old_images]
|
||||
|
||||
if old_image_ids:
|
||||
Annotation.query.filter(Annotation.image_id.in_(old_image_ids)).delete(synchronize_session=False)
|
||||
Image.query.filter_by(project_id=project['id']).delete()
|
||||
db.session.commit()
|
||||
print(f"Deleted {len(old_image_ids)} old images and their annotations for project {project['id']}")
|
||||
|
||||
# Prepare arrays
|
||||
images_bulk = []
|
||||
anns_bulk = []
|
||||
|
||||
for ann in data:
|
||||
# Extract width/height
|
||||
width = None
|
||||
height = None
|
||||
|
||||
if isinstance(ann.get('label_rectangles'), list) and len(ann['label_rectangles']) > 0:
|
||||
width = ann['label_rectangles'][0].get('original_width')
|
||||
height = ann['label_rectangles'][0].get('original_height')
|
||||
elif isinstance(ann.get('label'), list) and len(ann['label']) > 0:
|
||||
if ann['label'][0].get('original_width') and ann['label'][0].get('original_height'):
|
||||
width = ann['label'][0]['original_width']
|
||||
height = ann['label'][0]['original_height']
|
||||
|
||||
# Only process if width and height are valid
|
||||
if width and height:
|
||||
image_data = {
|
||||
'project_id': project['id'],
|
||||
'image_path': ann.get('image'),
|
||||
'width': int(width), # Ensure integer
|
||||
'height': int(height) # Ensure integer
|
||||
}
|
||||
images_bulk.append(image_data)
|
||||
|
||||
# Handle multiple annotations per image
|
||||
if isinstance(ann.get('label_rectangles'), list):
|
||||
for ann_detail in ann['label_rectangles']:
|
||||
# Get label safely
|
||||
rectanglelabels = ann_detail.get('rectanglelabels', [])
|
||||
if isinstance(rectanglelabels, list) and len(rectanglelabels) > 0:
|
||||
label = rectanglelabels[0]
|
||||
elif isinstance(rectanglelabels, str):
|
||||
label = rectanglelabels
|
||||
else:
|
||||
label = 'unknown'
|
||||
|
||||
ann_data = {
|
||||
'image_path': ann.get('image'),
|
||||
'x': (ann_detail['x'] * width) / 100,
|
||||
'y': (ann_detail['y'] * height) / 100,
|
||||
'width': (ann_detail['width'] * width) / 100,
|
||||
'height': (ann_detail['height'] * height) / 100,
|
||||
'Label': label
|
||||
}
|
||||
anns_bulk.append(ann_data)
|
||||
elif isinstance(ann.get('label'), list):
|
||||
for ann_detail in ann['label']:
|
||||
# Get label safely
|
||||
rectanglelabels = ann_detail.get('rectanglelabels', [])
|
||||
if isinstance(rectanglelabels, list) and len(rectanglelabels) > 0:
|
||||
label = rectanglelabels[0]
|
||||
elif isinstance(rectanglelabels, str):
|
||||
label = rectanglelabels
|
||||
else:
|
||||
label = 'unknown'
|
||||
|
||||
ann_data = {
|
||||
'image_path': ann.get('image'),
|
||||
'x': (ann_detail['x'] * width) / 100,
|
||||
'y': (ann_detail['y'] * height) / 100,
|
||||
'width': (ann_detail['width'] * width) / 100,
|
||||
'height': (ann_detail['height'] * height) / 100,
|
||||
'Label': label
|
||||
}
|
||||
anns_bulk.append(ann_data)
|
||||
|
||||
# Insert images and get generated IDs
|
||||
inserted_images = []
|
||||
for img_data in images_bulk:
|
||||
new_image = Image(**img_data)
|
||||
db.session.add(new_image)
|
||||
db.session.flush() # Flush to get the ID
|
||||
inserted_images.append(new_image)
|
||||
db.session.commit()
|
||||
|
||||
# Map image_path -> image_id
|
||||
image_map = {img.image_path: img.image_id for img in inserted_images}
|
||||
|
||||
# Assign correct image_id to each annotation
|
||||
for ann_data in anns_bulk:
|
||||
ann_data['image_id'] = image_map.get(ann_data['image_path'])
|
||||
del ann_data['image_path']
|
||||
|
||||
# Insert annotations
|
||||
for ann_data in anns_bulk:
|
||||
new_annotation = Annotation(**ann_data)
|
||||
db.session.add(new_annotation)
|
||||
db.session.commit()
|
||||
|
||||
print(f"Inserted {len(images_bulk)} images and {len(anns_bulk)} annotations for project {project['id']}")
|
||||
|
||||
print('Seeding done')
|
||||
return {'success': True, 'message': 'Data inserted successfully!'}
|
||||
|
||||
except Exception as error:
|
||||
print(f'Error inserting data: {error}')
|
||||
db.session.rollback()
|
||||
return {'success': False, 'message': str(error)}
|
||||
|
||||
finally:
|
||||
update_status["running"] = False
|
||||
print('updateStatus.running set to false')
|
||||
|
||||
71
backend/services/settings_service.py
Normal file
71
backend/services/settings_service.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Settings Service - Manages global application settings
|
||||
"""
|
||||
from models.Settings import Settings
|
||||
from database.database import db
|
||||
|
||||
def get_setting(key, default=None):
|
||||
"""Get a setting value by key"""
|
||||
setting = Settings.query.filter_by(key=key).first()
|
||||
return setting.value if setting else default
|
||||
|
||||
def set_setting(key, value, description=None):
|
||||
"""Set a setting value"""
|
||||
setting = Settings.query.filter_by(key=key).first()
|
||||
if setting:
|
||||
setting.value = value
|
||||
if description:
|
||||
setting.description = description
|
||||
else:
|
||||
setting = Settings(key=key, value=value, description=description)
|
||||
db.session.add(setting)
|
||||
|
||||
db.session.commit()
|
||||
return setting
|
||||
|
||||
def get_all_settings():
|
||||
"""Get all settings as a dictionary"""
|
||||
settings = Settings.query.all()
|
||||
return {s.key: s.value for s in settings}
|
||||
|
||||
def get_all_settings_detailed():
|
||||
"""Get all settings with full details"""
|
||||
settings = Settings.query.all()
|
||||
return [s.to_dict() for s in settings]
|
||||
|
||||
def initialize_default_settings():
|
||||
"""Initialize default settings if they don't exist"""
|
||||
defaults = {
|
||||
'labelstudio_api_url': {
|
||||
'value': 'http://192.168.1.19:8080/api',
|
||||
'description': 'Label Studio API URL'
|
||||
},
|
||||
'labelstudio_api_token': {
|
||||
'value': 'c1cef980b7c73004f4ee880a42839313b863869f',
|
||||
'description': 'Label Studio API Token'
|
||||
},
|
||||
'yolox_path': {
|
||||
'value': 'C:/YOLOX',
|
||||
'description': 'Path to YOLOX installation directory'
|
||||
},
|
||||
'yolox_venv_path': {
|
||||
'value': '/home/kitraining/Yolox/yolox_venv/bin/activate',
|
||||
'description': 'Path to YOLOX virtual environment activation script'
|
||||
},
|
||||
'yolox_output_path': {
|
||||
'value': './backend',
|
||||
'description': 'Output folder for YOLOX experiment files and JSONs'
|
||||
},
|
||||
'yolox_data_dir': {
|
||||
'value': '/home/kitraining/To_Annotate/',
|
||||
'description': 'Data directory path for YOLOX training (where images are located)'
|
||||
}
|
||||
}
|
||||
|
||||
for key, data in defaults.items():
|
||||
existing = Settings.query.filter_by(key=key).first()
|
||||
if not existing:
|
||||
setting = Settings(key=key, value=data['value'], description=data['description'])
|
||||
db.session.add(setting)
|
||||
|
||||
db.session.commit()
|
||||
Reference in New Issue
Block a user