Gurobi parallel load COBRA (.mat) model
Hi,
I am trying to load about 300 .mat model of COBRA parallel, I used the below function:
def get_capacity_matrix_parallel(model_path, target_rxns, AD_samples, HT_samples,
n_workers=16):
all_fnames = sorted(os.listdir(model_path))
all_fnames = [f for f in all_fnames if f.endswith('.mat')]
n_models = len(all_fnames)
if n_models == 0:
raise RuntimeError(f"No .mat files found in {model_path}")
n_workers = min(n_workers, n_models)
AD_set = set(AD_samples)
HT_set = set(HT_samples)
matrix = np.zeros((n_models, len(target_rxns)))
phenotypes = [None] * n_models
model_dict = {'AD': {}, 'HT': {}}
fname_to_idx = {f: i for i, f in enumerate(all_fnames)}
args_list = [
(fname, model_path, target_rxns, AD_set, HT_set)
for fname in all_fnames
]
errors = []
print(f"Loading {n_models} models with {n_workers} workers "
f"(each worker: own Gurobi Env, Threads=1)")
t0 = time.time()
with Pool(processes=n_workers, initializer=_pool_initializer) as pool:
for result in tqdm(
pool.imap_unordered(_load_and_compute_one, args_list, chunksize=1),
total=n_models, desc="Models",
):
idx = fname_to_idx[result['fname']]
matrix[idx] = result['v_max_row']
phenotypes[idx] = result['phenotype']
if result['error']:
errors.append((result['fname'], result['error']))
continue
if result['phenotype'] in ('AD', 'HT') and result['model'] is not None:
model_dict[result['phenotype']][result['sample_id']] = result['model']
elapsed = time.time() - t0
print(f"\nFinished in {elapsed:.1f}s ({elapsed/n_models:.2f}s per model avg)")
if errors:
print(f"\n{len(errors)} models failed:")
for fname, err in errors[:10]:
print(f" {fname}: {err}")
if len(errors) > 10:
print(f" ... and {len(errors) - 10} more")
return matrix, model_dict, all_fnames, phenotypes, n_models But it failed to have parallel loading processes, still load the .mat model serially. Could you please give me some guidance on what should I do? Thanks!
0
Please sign in to leave a comment.
Comments
0 comments