guanjihuan/py.guanjihuan.com
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
guanjihuan 2025-03-23 01:52:08 +08:00
parent 6ec38a3e08
commit e173828a2d
2 changed files with 58 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PyPI/setup.cfg
View File

@ -1,7 +1,7 @@
[metadata]
# replace with your username:
name = guan
version = 0.1.168
version = 0.1.169
author = guanjihuan
author_email = guanjihuan@163.com
description = An open source python package

59
PyPI/src/guan/data_processing.py
View File

@ -125,13 +125,26 @@ def make_sh_file_for_qsub(sh_filename='a', command_line='python a.py', cpu_num=1
with open(sh_filename+'.sh', 'w') as f:
f.write(sh_content)
# 创建一个sh文件用于提交任务（Slurm）
def make_sh_file_for_sbatch(sh_filename='a', command_line='python a.py', cpu_num=1, task_name='task', cd_dir=0):
sh_content = \
'#!/bin/sh\n' \
+'#SBATCH --job-name='+task_name+'\n' \
+'#SBATCH --cpus-per-task='+str(cpu_num)+'\n'
if cd_dir==1:
sh_content += 'cd $PBS_O_WORKDIR\n'
sh_content += command_line
with open(sh_filename+'.sh', 'w') as f:
f.write(sh_content)
# 创建一个sh文件用于提交任务（LSF）
def make_sh_file_for_bsub(sh_filename='a', command_line='python a.py', cpu_num=1, task_name='task', queue_name='score', cd_dir=0):
def make_sh_file_for_bsub(sh_filename='a', command_line='python a.py', cpu_num=1, task_name='task', cd_dir=0, bsub_q=0, queue_name='score'):
sh_content = \
'#!/bin/sh\n' \
+'#BSUB -J '+task_name+'\n' \
+'#BSUB -q '+queue_name+'\n' \
+'#BSUB -n '+str(cpu_num)+'\n'
if bsub_q==1:
sh_content += '#BSUB -q '+queue_name+'\n'
if cd_dir==1:
sh_content += 'cd $PBS_O_WORKDIR\n'
sh_content += command_line
@ -143,6 +156,11 @@ def qsub_task(filename='a', file_format='.sh'):
import os
os.system('qsub '+filename+file_format)
# sbatch 提交任务（Slurm）
def sbatch_task(filename='a', file_format='.sh'):
import os
os.system('sbatch '+filename+file_format)
# bsub 提交任务（LSF）
def bsub_task(filename='a', file_format='.sh'):
import os
@ -185,6 +203,43 @@ def copy_py_sh_file_and_qsub_task(parameter_array, py_filename='a', old_str_in_p
# qsub task
os.system('qsub '+new_file)
# 复制.py和.sh文件，然后提交任务，实现半手动并行（Slurm）
def copy_py_sh_file_and_sbatch_task(parameter_array, py_filename='a', old_str_in_py='parameter = 0', new_str_in_py='parameter = ', sh_filename='a', task_name='task'):
import os
parameter_str_array = []
for i0 in parameter_array:
parameter_str_array.append(str(i0))
index = 0
for parameter_str in parameter_str_array:
index += 1
# copy python file
old_file = py_filename+'.py'
new_file = py_filename+'_'+str(index)+'.py'
os.system('cp '+old_file+' '+new_file)
with open(new_file, 'r') as f:
content = f.read()
old_str = old_str_in_py
new_str = new_str_in_py+parameter_str
content = content.replace(old_str, new_str)
with open(py_filename+'_'+str(index)+'.py', 'w') as f:
f.write(content)
# copy sh file
old_file = sh_filename+'.sh'
new_file = sh_filename+'_'+str(index)+'.sh'
os.system('cp '+old_file+' '+new_file)
with open(new_file, 'r') as f:
content = f.read()
old_str = 'python '+py_filename+'.py'
new_str = 'python '+py_filename+'_'+str(index)+'.py'
content = content.replace(old_str, new_str)
old_str = '#SBATCH --job-name='+task_name
new_str = '#SBATCH --job-name='+task_name+'_'+str(index)
content = content.replace(old_str, new_str)
with open(sh_filename+'_'+str(index)+'.sh', 'w') as f:
f.write(content)
# sbatch task
os.system('sbatch '+new_file)
# 复制.py和.sh文件，然后提交任务，实现半手动并行（LSF）
def copy_py_sh_file_and_bsub_task(parameter_array, py_filename='a', old_str_in_py='parameter = 0', new_str_in_py='parameter = ', sh_filename='a', task_name='task'):
import os