diff --git a/PyPI/setup.cfg b/PyPI/setup.cfg index dc4c87a..79eb742 100644 --- a/PyPI/setup.cfg +++ b/PyPI/setup.cfg @@ -1,7 +1,7 @@ [metadata] # replace with your username: name = guan -version = 0.1.168 +version = 0.1.169 author = guanjihuan author_email = guanjihuan@163.com description = An open source python package diff --git a/PyPI/src/guan/data_processing.py b/PyPI/src/guan/data_processing.py index 9609426..e9acf29 100644 --- a/PyPI/src/guan/data_processing.py +++ b/PyPI/src/guan/data_processing.py @@ -125,13 +125,26 @@ def make_sh_file_for_qsub(sh_filename='a', command_line='python a.py', cpu_num=1 with open(sh_filename+'.sh', 'w') as f: f.write(sh_content) +# 创建一个sh文件用于提交任务(Slurm) +def make_sh_file_for_sbatch(sh_filename='a', command_line='python a.py', cpu_num=1, task_name='task', cd_dir=0): + sh_content = \ + '#!/bin/sh\n' \ + +'#SBATCH --job-name='+task_name+'\n' \ + +'#SBATCH --cpus-per-task='+str(cpu_num)+'\n' + if cd_dir==1: + sh_content += 'cd $PBS_O_WORKDIR\n' + sh_content += command_line + with open(sh_filename+'.sh', 'w') as f: + f.write(sh_content) + # 创建一个sh文件用于提交任务(LSF) -def make_sh_file_for_bsub(sh_filename='a', command_line='python a.py', cpu_num=1, task_name='task', queue_name='score', cd_dir=0): +def make_sh_file_for_bsub(sh_filename='a', command_line='python a.py', cpu_num=1, task_name='task', cd_dir=0, bsub_q=0, queue_name='score'): sh_content = \ '#!/bin/sh\n' \ +'#BSUB -J '+task_name+'\n' \ - +'#BSUB -q '+queue_name+'\n' \ +'#BSUB -n '+str(cpu_num)+'\n' + if bsub_q==1: + sh_content += '#BSUB -q '+queue_name+'\n' if cd_dir==1: sh_content += 'cd $PBS_O_WORKDIR\n' sh_content += command_line @@ -143,6 +156,11 @@ def qsub_task(filename='a', file_format='.sh'): import os os.system('qsub '+filename+file_format) +# sbatch 提交任务(Slurm) +def sbatch_task(filename='a', file_format='.sh'): + import os + os.system('sbatch '+filename+file_format) + # bsub 提交任务(LSF) def bsub_task(filename='a', file_format='.sh'): import os @@ -185,6 +203,43 @@ def copy_py_sh_file_and_qsub_task(parameter_array, py_filename='a', old_str_in_p # qsub task os.system('qsub '+new_file) +# 复制.py和.sh文件,然后提交任务,实现半手动并行(Slurm) +def copy_py_sh_file_and_sbatch_task(parameter_array, py_filename='a', old_str_in_py='parameter = 0', new_str_in_py='parameter = ', sh_filename='a', task_name='task'): + import os + parameter_str_array = [] + for i0 in parameter_array: + parameter_str_array.append(str(i0)) + index = 0 + for parameter_str in parameter_str_array: + index += 1 + # copy python file + old_file = py_filename+'.py' + new_file = py_filename+'_'+str(index)+'.py' + os.system('cp '+old_file+' '+new_file) + with open(new_file, 'r') as f: + content = f.read() + old_str = old_str_in_py + new_str = new_str_in_py+parameter_str + content = content.replace(old_str, new_str) + with open(py_filename+'_'+str(index)+'.py', 'w') as f: + f.write(content) + # copy sh file + old_file = sh_filename+'.sh' + new_file = sh_filename+'_'+str(index)+'.sh' + os.system('cp '+old_file+' '+new_file) + with open(new_file, 'r') as f: + content = f.read() + old_str = 'python '+py_filename+'.py' + new_str = 'python '+py_filename+'_'+str(index)+'.py' + content = content.replace(old_str, new_str) + old_str = '#SBATCH --job-name='+task_name + new_str = '#SBATCH --job-name='+task_name+'_'+str(index) + content = content.replace(old_str, new_str) + with open(sh_filename+'_'+str(index)+'.sh', 'w') as f: + f.write(content) + # sbatch task + os.system('sbatch '+new_file) + # 复制.py和.sh文件,然后提交任务,实现半手动并行(LSF) def copy_py_sh_file_and_bsub_task(parameter_array, py_filename='a', old_str_in_py='parameter = 0', new_str_in_py='parameter = ', sh_filename='a', task_name='task'): import os