Comment mettre en œuvre le script python pour s'exécuter sur le nombre "N" de CPU CORES?

Question

J'ai créé un script pour optimiser une partie de la structure. (termes scientifiques, vous pouvez l'ignorer) mais le but principal du script est l'optimisation et cela prend beaucoup de temps au cours de ces deux étapes optimiser() et affiner() fonction où il utilise seulement un processeur sur 4 CPU dans mon système local mais vouloir faire ce script pour utiliser tous les 4 CPU (en particulier pour ces deux fonctions optimize() et affine()).

Je n'avais pas beaucoup d'idée sur le multiprocessing/multicore mais je continue d'utiliser un module de multiprocessing mais il échoue à utiliser tous les processeurs. Donc, si quelqu'un sait comment mettre en œuvre le script python pour fonctionner à tous les avantages multiples CPU pourrait me donner une suggestion serait vraiment utile.

MY SCRIPT:

import sys 
import os 

from modeller import * 
from modeller.optimizers import molecular_dynamics,conjugate_gradients 
from modeller.automodel import autosched 

def optimize(atmsel, sched): 
    for step in sched: 
     step.optimize(atmsel, max_iterations=200, min_atom_shift=0.001) 
    refine(atmsel) 
    cg = conjugate_gradients() 
    cg.optimize(atmsel, max_iterations=200, min_atom_shift=0.001) 

def refine(atmsel): 
    md = molecular_dynamics(cap_atom_shift=0.39, md_time_step=4.0, 
         md_return='FINAL') 
    init_vel = True 
    for (its, equil, temps) in ((200, 20, (150.0, 250.0, 400.0, 700.0, 1000.0)), 
          (200, 600, 
          (1000.0, 800.0, 600.0, 500.0, 400.0, 300.0))): 
    for temp in temps: 
     md.optimize(atmsel, init_velocities=init_vel, temperature=temp, 
        max_iterations=its, equilibrate=equil) 
     init_vel = False 

def make_restraints(mdl1, aln): 
    rsr = mdl1.restraints 
    rsr.clear() 
    s = selection(mdl1) 
    for typ in ('stereo', 'phi-psi_binormal'): 
     rsr.make(s, restraint_type=typ, aln=aln, spline_on_site=True) 
    for typ in ('omega', 'chi1', 'chi2', 'chi3', 'chi4'): 
     rsr.make(s, restraint_type=typ+'_dihedral', spline_range=4.0, 
      spline_dx=0.3, spline_min_points = 5, aln=aln, 
      spline_on_site=True) 

log.verbose() 

env = environ(rand_seed=int(-4243)) 
env.io.hetatm = True 
env.edat.dynamic_sphere=False 
env.edat.dynamic_lennard=True 
env.edat.contact_shell = 4.0 
env.edat.update_dynamic = 0.39 

env.libs.topology.read(file='$(LIB)/top_heav.lib') 

env.libs.parameters.read(file='$(LIB)/par.lib') 

mdl1 = model(env, file = "3O26") 
ali = alignment(env) 
ali.append_model(mdl1, atom_files= "3O26.pdb", align_codes= "3O26") 

s = selection(mdl1.chains["A"].residues["275"]) 

s.mutate(residue_type="ALA") 
ali.append_model(mdl1, align_codes="3O26") 

mdl1.clear_topology() 
mdl1.generate_topology(ali[-1]) 

mdl1.transfer_xyz(ali) 

mdl1.build(initialize_xyz=False, build_method='INTERNAL_COORDINATES') 

mdl2 = model(env, file="3O26.pdb") 

mdl1.res_num_from(mdl2,ali) 

mdl1.write(file="3O26"+"ALA"+"275"+"A"+'.tmp') 
mdl1.read(file="3O26"+"ALA"+"275"+"A"+'.tmp') 

make_restraints(mdl1, ali) 
mdl1.env.edat.nonbonded_sel_atoms=1 
sched = autosched.loop.make_for_model(mdl1) 

s = selection(mdl1.atoms['CA:'+"275"+':'+"A"].select_sphere(5)).by_residue() 

mdl1.restraints.unpick_all() 
mdl1.restraints.pick(s) 

s.energy() 

s.randomize_xyz(deviation=4.0) 

mdl1.env.edat.nonbonded_sel_atoms=2 
optimize(s,sched) 
mdl1.env.edat.nonbonded_sel_atoms=1 
optimize(s,sched) 
s.energy() 
atmsel = selection(mdl1.chains["A"]) 
score = atmsel.assess_dope() 
mdl1.write(file="hi.pdb") 

os.remove("3O26"+"ALA"+"275"+"A"+'.tmp') 

from multiprocessing import Process 
if __name__ == '__main__': 
    p = Process(target=optimize, args=(atmsel,sched)) 
    p.start() 
    p.join() 

En cas de démonstration, de bien vouloir coller ce (http://files.rcsb.org/view/3o26.pdb) dans un fichier 3O26.pdb et conservez-le dans le même répertoire.

vous remerciant à l'avance

Sur la base de suggestion @Dinesh j'ai modifié le code en incluant le module de pages où son travail très bien avec l'aide de tous les noyaux mais je reçois des erreurs que je ne pouvais pas comprendre.

script modifié:

import sys 
import os 
import pp 
from modeller import * 
from modeller.optimizers import molecular_dynamics, conjugate_gradients 
from modeller.automodel import autosched 


def optimize(atmsel, sched): 
    for step in sched: 
     step.optimize(atmsel, max_iterations=200, min_atom_shift=0.001) 
    refine(atmsel) 
    cg = conjugate_gradients() 
    cg.optimize(atmsel, max_iterations=200, min_atom_shift=0.001) 


def refine(atmsel): 
    md = molecular_dynamics(cap_atom_shift=0.39, md_time_step=4.0, 
          md_return='FINAL') 
    init_vel = True 
    for (its, equil, temps) in ((200, 20, (150.0, 250.0, 400.0, 700.0, 1000.0)), 
           (200, 600, 
           (1000.0, 800.0, 600.0, 500.0, 400.0, 300.0))): 
     for temp in temps: 
      md.optimize(atmsel, init_velocities=init_vel, temperature=temp, 
         max_iterations=its, equilibrate=equil) 
      init_vel = False 

def make_restraints(mdl1, aln): 
    rsr = mdl1.restraints 
    rsr.clear() 
    s = selection(mdl1) 
    for typ in ('stereo', 'phi-psi_binormal'): 
     rsr.make(s, restraint_type=typ, aln=aln, spline_on_site=True) 
    for typ in ('omega', 'chi1', 'chi2', 'chi3', 'chi4'): 
     rsr.make(s, restraint_type=typ + '_dihedral', spline_range=4.0, 
       spline_dx=0.3, spline_min_points=5, aln=aln, 
       spline_on_site=True) 

################################### PPMODULE ############################ 
def main(s,sched): 
    print s,"*************************************************************************" 
    ppservers =() 
    if len(sys.argv) > 1: 
     ncpus = int(sys.argv[1]) 
     job_server = pp.Server(ncpus, ppservers=ppservers) 
    else: 
     job_server = pp.Server(ppservers=ppservers) 
    print "Starting pp with", job_server.get_ncpus(), "workers" 
    job_server.submit(optimize,(s,sched,),(refine,),("from modeller.optimizers import molecular_dynamics, conjugate_gradients",))() 
#################################### PPMODULE ############################ 

if __name__=="__main__": 
    log.verbose() 
    env = environ(rand_seed=int(-4345)) 
    env.io.hetatm = True 
    env.edat.dynamic_sphere = False 
    env.edat.dynamic_lennard = True 
    env.edat.contact_shell = 4.0 
    env.edat.update_dynamic = 0.39 
    env.libs.topology.read(file='$(LIB)/top_heav.lib') 
    env.libs.parameters.read(file='$(LIB)/par.lib') 
    mdl1 = model(env, file="3O26") 
    ali = alignment(env) 
    ali.append_model(mdl1, atom_files="3O26.pdb", align_codes="3O26") 
    s = selection(mdl1.chains["A"].residues["275"]) 
    s.mutate(residue_type="ALA") 
    ali.append_model(mdl1, align_codes="3O26") 
    mdl1.clear_topology() 
    mdl1.generate_topology(ali[-1]) 
    mdl1.transfer_xyz(ali) 
    mdl1.build(initialize_xyz=False, build_method='INTERNAL_COORDINATES') 
    mdl2 = model(env, file="3O26.pdb") 
    mdl1.res_num_from(mdl2, ali) 
    mdl1.write(file="3O26" + "ALA" + "275" + "A" + '.tmp') 
    mdl1.read(file="3O26" + "ALA" + "275" + "A" + '.tmp') 
    make_restraints(mdl1, ali) 
    mdl1.env.edat.nonbonded_sel_atoms = 1 
    sched = autosched.loop.make_for_model(mdl1) 
    s = selection(mdl1.atoms['CA:' + "275" + ':' + "A"].select_sphere(15)).by_residue() 
    mdl1.restraints.unpick_all() 
    mdl1.restraints.pick(s) 
    s.energy() 
    s.randomize_xyz(deviation=4.0) 
    mdl1.env.edat.nonbonded_sel_atoms = 2 
    main(s, sched) 
    mdl1.env.edat.nonbonded_sel_atoms = 1 
    main(s, sched) 
    s.energy() 
    atmsel = selection(mdl1.chains["A"]) 
    score = atmsel.assess_dope() 
    mdl1.write(file="current.pdb") 
    os.remove("3O26" + "ALA" + "275" + "A" + '.tmp') 

ERREUR:

randomi_498_> Atoms,selected atoms,random_seed,amplitude:  2302  558  1  4.0000 
randomi_496_> Amplitude is > 0; randomization is done. 
<Selection of 558 atoms> ************************************************************************* 
Starting pp with 4 workers 
Traceback (most recent call last): 
    File "mutate_model.py", line 88, in <module> 
    main(s, sched) 
    File "m_m.py", line 52, in main 
    job_server.submit(optimize,(s,sched,),(refine,),("from modeller.optimizers import molecular_dynamics, conjugate_gradients",))() 
    File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pp.py", line 460, in submit 
    sfunc = self.__dumpsfunc((func,) + depfuncs, modules) 
    File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pp.py", line 638, in __dumpsfunc 
    sources = [self.__get_source(func) for func in funcs] 
    File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pp.py", line 705, in __get_source 
    sourcelines = inspect.getsourcelines(func)[0] 
    File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/inspect.py", line 690, in getsourcelines 
    lines, lnum = findsource(object) 
    File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/inspect.py", line 526, in findsource 
    file = getfile(object) 
    File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/inspect.py", line 408, in getfile 
    raise TypeError('{!r} is a built-in class'.format(object)) 
TypeError: <module '__builtin__' (built-in)> is a built-in class 

Answer 1

Enfin je me l'ont fait par une autre méthode appelée multiprocessing.pool basée sur le blog http://chriskiehl.com/article/parallelism-in-one-line/ et https://pymotw.com/2/multiprocessing/basics.html

Voici mon code Pseudo:

from multiprocessing import Pool 

def get_mm_script(scripts): 
    #I just created all my mm.py scripts as string template 
    return scripts 

def run(filename): 
    #here i use system command to run my all scripts. 
    return 

if __name__ == '__main__': 
    scripts=get_mm_script(f) 
    pool = Pool(4) 
    pool.map(run, scripts) 
    pool.close() 
    pool.join() 

Answer 2

S'il vous plaît vérifier le module pp. Parallelpython.com