implemented locking in training setup
This commit is contained in:
Philipp
@@ -1,8 +1,31 @@
|
||||
import os
|
||||
import shutil
|
||||
import importlib.util
|
||||
from models.training import Training
|
||||
from models.TrainingProject import TrainingProject
|
||||
|
||||
def load_base_config(selected_model):
|
||||
"""Load base configuration for a specific YOLOX model"""
|
||||
model_name = selected_model.lower().replace('-', '_').replace('.pth', '')
|
||||
base_config_path = os.path.join(os.path.dirname(__file__), '..', 'data', f'{model_name}.py')
|
||||
|
||||
if not os.path.exists(base_config_path):
|
||||
raise Exception(f'Base configuration not found for model: {model_name} at {base_config_path}')
|
||||
|
||||
# Load the module dynamically
|
||||
spec = importlib.util.spec_from_file_location(f"base_config_{model_name}", base_config_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
# Extract all attributes from BaseExp class
|
||||
base_exp = module.BaseExp()
|
||||
base_config = {}
|
||||
for attr in dir(base_exp):
|
||||
if not attr.startswith('_'):
|
||||
base_config[attr] = getattr(base_exp, attr)
|
||||
|
||||
return base_config
|
||||
|
||||
def generate_yolox_exp(training_id):
|
||||
"""Generate YOLOX exp.py file"""
|
||||
# Fetch training row from DB
|
||||
@@ -13,26 +36,14 @@ def generate_yolox_exp(training_id):
|
||||
if not training:
|
||||
raise Exception(f'Training not found for trainingId or project_details_id: {training_id}')
|
||||
|
||||
# If transfer_learning is 'coco', copy default exp.py
|
||||
# If transfer_learning is 'coco', generate exp using base config + custom settings
|
||||
if training.transfer_learning == 'coco':
|
||||
selected_model = training.selected_model.lower().replace('-', '_')
|
||||
exp_source_path = f'/home/kitraining/Yolox/YOLOX-main/exps/default/{selected_model}.py'
|
||||
|
||||
if not os.path.exists(exp_source_path):
|
||||
raise Exception(f'Default exp.py not found for model: {selected_model} at {exp_source_path}')
|
||||
|
||||
# Copy to project folder
|
||||
project_details_id = training.project_details_id
|
||||
project_folder = os.path.join(os.path.dirname(__file__), '..', f'project_23/{project_details_id}')
|
||||
os.makedirs(project_folder, exist_ok=True)
|
||||
|
||||
exp_dest_path = os.path.join(project_folder, 'exp.py')
|
||||
shutil.copyfile(exp_source_path, exp_dest_path)
|
||||
return {'type': 'default', 'expPath': exp_dest_path}
|
||||
exp_content = generate_yolox_inference_exp(training_id, use_base_config=True)
|
||||
return {'type': 'custom', 'expContent': exp_content}
|
||||
|
||||
# If transfer_learning is 'sketch', generate custom exp.py
|
||||
if training.transfer_learning == 'sketch':
|
||||
exp_content = generate_yolox_inference_exp(training_id)
|
||||
exp_content = generate_yolox_inference_exp(training_id, use_base_config=False)
|
||||
return {'type': 'custom', 'expContent': exp_content}
|
||||
|
||||
raise Exception(f'Unknown transfer_learning type: {training.transfer_learning}')
|
||||
@@ -53,8 +64,14 @@ def save_yolox_exp(training_id, out_path):
|
||||
else:
|
||||
raise Exception('Unknown expResult type or missing content')
|
||||
|
||||
def generate_yolox_inference_exp(training_id, options=None):
|
||||
"""Generate inference exp.py using DB values"""
|
||||
def generate_yolox_inference_exp(training_id, options=None, use_base_config=False):
|
||||
"""Generate inference exp.py using DB values
|
||||
|
||||
Args:
|
||||
training_id: The training/project_details ID
|
||||
options: Optional overrides for data paths
|
||||
use_base_config: If True, load base config and only override with user-defined values
|
||||
"""
|
||||
if options is None:
|
||||
options = {}
|
||||
|
||||
@@ -90,14 +107,69 @@ def generate_yolox_inference_exp(training_id, options=None):
|
||||
except Exception as e:
|
||||
print(f'Could not determine num_classes from TrainingProject.classes: {e}')
|
||||
|
||||
depth = options.get('depth', training.depth or 1.00)
|
||||
width = options.get('width', training.width or 1.00)
|
||||
input_size = options.get('input_size', training.input_size or [640, 640])
|
||||
mosaic_scale = options.get('mosaic_scale', training.mosaic_scale or [0.1, 2])
|
||||
random_size = options.get('random_size', [10, 20])
|
||||
test_size = options.get('test_size', training.test_size or [640, 640])
|
||||
exp_name = options.get('exp_name', 'inference_exp')
|
||||
enable_mixup = options.get('enable_mixup', False)
|
||||
# Initialize config dictionary
|
||||
config = {}
|
||||
|
||||
# If using base config (transfer learning from COCO), load protected parameters first
|
||||
if use_base_config and training.selected_model:
|
||||
try:
|
||||
base_config = load_base_config(training.selected_model)
|
||||
config.update(base_config)
|
||||
print(f'Loaded base config for {training.selected_model}: {list(base_config.keys())}')
|
||||
except Exception as e:
|
||||
print(f'Warning: Could not load base config for {training.selected_model}: {e}')
|
||||
print('Falling back to custom settings only')
|
||||
|
||||
# Override with user-defined values from training table (only if they exist and are not None)
|
||||
user_overrides = {
|
||||
'depth': training.depth,
|
||||
'width': training.width,
|
||||
'input_size': training.input_size,
|
||||
'mosaic_scale': training.mosaic_scale,
|
||||
'test_size': training.test_size,
|
||||
'enable_mixup': training.enable_mixup,
|
||||
'max_epoch': training.max_epoch,
|
||||
'warmup_epochs': training.warmup_epochs,
|
||||
'warmup_lr': training.warmup_lr,
|
||||
'basic_lr_per_img': training.basic_lr_per_img,
|
||||
'scheduler': training.scheduler,
|
||||
'no_aug_epochs': training.no_aug_epochs,
|
||||
'min_lr_ratio': training.min_lr_ratio,
|
||||
'ema': training.ema,
|
||||
'weight_decay': training.weight_decay,
|
||||
'momentum': training.momentum,
|
||||
'print_interval': training.print_interval,
|
||||
'eval_interval': training.eval_interval,
|
||||
'test_conf': training.test_conf,
|
||||
'nms_thre': training.nms_thre,
|
||||
'mosaic_prob': training.mosaic_prob,
|
||||
'mixup_prob': training.mixup_prob,
|
||||
'hsv_prob': training.hsv_prob,
|
||||
'flip_prob': training.flip_prob,
|
||||
'degrees': training.degrees,
|
||||
'translate': training.translate,
|
||||
'shear': training.shear,
|
||||
'mixup_scale': training.mixup_scale,
|
||||
'activation': training.activation,
|
||||
}
|
||||
|
||||
# Only override if value is explicitly set (not None)
|
||||
for key, value in user_overrides.items():
|
||||
if value is not None:
|
||||
config[key] = value
|
||||
|
||||
# Apply any additional options overrides
|
||||
config.update(options)
|
||||
|
||||
# Set defaults for any missing required parameters
|
||||
config.setdefault('depth', 1.00)
|
||||
config.setdefault('width', 1.00)
|
||||
config.setdefault('input_size', [640, 640])
|
||||
config.setdefault('mosaic_scale', [0.1, 2])
|
||||
config.setdefault('random_size', [10, 20])
|
||||
config.setdefault('test_size', [640, 640])
|
||||
config.setdefault('enable_mixup', False)
|
||||
config.setdefault('exp_name', 'inference_exp')
|
||||
|
||||
# Build exp content
|
||||
exp_content = f'''#!/usr/bin/env python3
|
||||
@@ -115,7 +187,7 @@ class Exp(MyExp):
|
||||
self.data_dir = "{data_dir}"
|
||||
self.train_ann = "{train_ann}"
|
||||
self.val_ann = "{val_ann}"
|
||||
self.test_ann = "coco_project_{training_id}_test.json"
|
||||
self.test_ann = "{test_ann}"
|
||||
self.num_classes = {num_classes}
|
||||
'''
|
||||
|
||||
@@ -127,26 +199,30 @@ class Exp(MyExp):
|
||||
exp_content += f" self.pretrained_ckpt = r'{yolox_base_dir}/pretrained/{selected_model}.pth'\n"
|
||||
|
||||
# Format arrays
|
||||
input_size_str = ', '.join(map(str, input_size)) if isinstance(input_size, list) else str(input_size)
|
||||
mosaic_scale_str = ', '.join(map(str, mosaic_scale)) if isinstance(mosaic_scale, list) else str(mosaic_scale)
|
||||
random_size_str = ', '.join(map(str, random_size)) if isinstance(random_size, list) else str(random_size)
|
||||
test_size_str = ', '.join(map(str, test_size)) if isinstance(test_size, list) else str(test_size)
|
||||
def format_value(val):
|
||||
if isinstance(val, (list, tuple)):
|
||||
return '(' + ', '.join(map(str, val)) + ')'
|
||||
elif isinstance(val, bool):
|
||||
return str(val)
|
||||
elif isinstance(val, str):
|
||||
return f'"{val}"'
|
||||
else:
|
||||
return str(val)
|
||||
|
||||
exp_content += f''' self.depth = {depth}
|
||||
self.width = {width}
|
||||
self.input_size = ({input_size_str})
|
||||
self.mosaic_scale = ({mosaic_scale_str})
|
||||
self.random_size = ({random_size_str})
|
||||
self.test_size = ({test_size_str})
|
||||
self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]
|
||||
self.enable_mixup = {str(enable_mixup)}
|
||||
# Add all config parameters to exp
|
||||
for key, value in config.items():
|
||||
if key not in ['exp_name']: # exp_name is handled separately
|
||||
exp_content += f" self.{key} = {format_value(value)}\n"
|
||||
|
||||
# Add exp_name at the end (uses dynamic path)
|
||||
exp_content += f''' self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0]
|
||||
'''
|
||||
|
||||
return exp_content
|
||||
|
||||
def save_yolox_inference_exp(training_id, out_path, options=None):
|
||||
"""Save inference exp.py to custom path"""
|
||||
exp_content = generate_yolox_inference_exp(training_id, options)
|
||||
exp_content = generate_yolox_inference_exp(training_id, options, use_base_config=False)
|
||||
with open(out_path, 'w') as f:
|
||||
f.write(exp_content)
|
||||
return out_path
|
||||
|
||||
@@ -5,32 +5,88 @@ from database.database import db
|
||||
def push_yolox_exp_to_db(settings):
|
||||
"""Save YOLOX settings to database"""
|
||||
normalized = dict(settings)
|
||||
|
||||
# Map 'act' from frontend to 'activation' for DB
|
||||
if 'act' in normalized:
|
||||
normalized['activation'] = normalized['act']
|
||||
del normalized['act']
|
||||
|
||||
# Convert 'on'/'off' to boolean for save_history_ckpt
|
||||
if isinstance(normalized.get('save_history_ckpt'), str):
|
||||
normalized['save_history_ckpt'] = normalized['save_history_ckpt'] == 'on'
|
||||
|
||||
# Convert comma-separated strings to arrays
|
||||
|
||||
# Map common frontend aliases to DB column names
|
||||
alias_map = {
|
||||
'act': 'activation',
|
||||
'nmsthre': 'nms_thre',
|
||||
'select_model': 'selected_model'
|
||||
}
|
||||
for a, b in alias_map.items():
|
||||
if a in normalized and b not in normalized:
|
||||
normalized[b] = normalized.pop(a)
|
||||
|
||||
# Convert 'on'/'off' or 'true'/'false' strings to boolean for known boolean fields
|
||||
for bool_field in ['save_history_ckpt', 'ema', 'enable_mixup']:
|
||||
if bool_field in normalized:
|
||||
val = normalized[bool_field]
|
||||
if isinstance(val, str):
|
||||
normalized[bool_field] = val.lower() in ('1', 'true', 'on')
|
||||
else:
|
||||
normalized[bool_field] = bool(val)
|
||||
|
||||
# Convert comma-separated strings to arrays for JSON fields
|
||||
for key in ['input_size', 'test_size', 'mosaic_scale', 'mixup_scale']:
|
||||
if isinstance(normalized.get(key), str):
|
||||
arr = [float(v.strip()) for v in normalized[key].split(',')]
|
||||
if key in normalized and isinstance(normalized[key], str):
|
||||
parts = [p.strip() for p in normalized[key].split(',') if p.strip()]
|
||||
try:
|
||||
arr = [float(p) for p in parts]
|
||||
except Exception:
|
||||
arr = parts
|
||||
normalized[key] = arr[0] if len(arr) == 1 else arr
|
||||
|
||||
# Find TrainingProjectDetails for this project
|
||||
details = TrainingProjectDetails.query.filter_by(project_id=normalized['project_id']).first()
|
||||
|
||||
# Ensure we have a TrainingProjectDetails row for project_id
|
||||
project_id = normalized.get('project_id')
|
||||
if not project_id:
|
||||
raise Exception('Missing project_id in settings')
|
||||
details = TrainingProjectDetails.query.filter_by(project_id=project_id).first()
|
||||
if not details:
|
||||
raise Exception(f'TrainingProjectDetails not found for project_id {normalized["project_id"]}')
|
||||
|
||||
raise Exception(f'TrainingProjectDetails not found for project_id {project_id}')
|
||||
normalized['project_details_id'] = details.id
|
||||
|
||||
|
||||
# Filter normalized to only columns that exist on the Training model
|
||||
valid_cols = {c.name: c for c in Training.__table__.columns}
|
||||
filtered = {}
|
||||
for k, v in normalized.items():
|
||||
if k in valid_cols:
|
||||
col_type = valid_cols[k].type.__class__.__name__
|
||||
# Try to coerce types for numeric/boolean columns
|
||||
try:
|
||||
if 'Integer' in col_type:
|
||||
if v is None or v == '':
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = int(float(v))
|
||||
elif 'Float' in col_type:
|
||||
if v is None or v == '':
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = float(v)
|
||||
elif 'Boolean' in col_type:
|
||||
if isinstance(v, str):
|
||||
filtered[k] = v.lower() in ('1', 'true', 'on')
|
||||
else:
|
||||
filtered[k] = bool(v)
|
||||
elif 'JSON' in col_type:
|
||||
filtered[k] = v
|
||||
elif 'LargeBinary' in col_type:
|
||||
# If a file path was passed, store its bytes; otherwise store raw bytes
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
filtered[k] = v.encode('utf-8')
|
||||
except Exception:
|
||||
filtered[k] = None
|
||||
else:
|
||||
filtered[k] = v
|
||||
else:
|
||||
filtered[k] = v
|
||||
except Exception:
|
||||
# If conversion fails, just assign raw value
|
||||
filtered[k] = v
|
||||
|
||||
# Create DB row
|
||||
training = Training(**normalized)
|
||||
training = Training(**filtered)
|
||||
db.session.add(training)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
return training
|
||||
|
||||
Reference in New Issue
Block a user