guanjihuan/guanjihuan.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
guanjihuan
2024-10-23 03:50:03 +08:00
parent 2d98dd5b07
commit 67d4c9e5a4
8 changed files with 0 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
2021.06.07_find_common_words_in_APS_abstracts/collect_prb_abstracts.py
View File

@@ -1,88 +0,0 @@
"""
This code is supported by the website: https://www.guanjihuan.com
The newest version of this code is on the web page: https://www.guanjihuan.com/archives/13623
"""
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
from collections import Counter
import datetime
import random
import time
# time.sleep(random.uniform(0,1800)) # 爬虫简单伪装，在固定时间后0到30分钟后开始运行。调试的时候把该语句注释。
year = datetime.datetime.now().year
month = datetime.datetime.now().month
day = datetime.datetime.now().day
# 获取链接
try:
with open('prb_link_list.txt', 'r', encoding='UTF-8') as f: # 如果文件存在
link_list = f.read().split('\n') # 历史已经访问过的链接（数组类型）
except:
with open('prb_link_list.txt', 'w', encoding='UTF-8') as f: # 如果文件不存在
link_list = []
f = open('prb_link_list.txt', 'a', encoding='UTF-8') # 打开文件（补充）
f.write('\nLink list obtained on '+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+':\n')
match_href = [] # 在本次运行中满足条件的链接
for loop in range(3):
if loop == 0:
start_link = "https://journals.aps.org/prb/recent?page=1" # 看第一页
elif loop == 1:
start_link = "https://journals.aps.org/prb/recent?page=2" # 看第二页
elif loop == 2:
start_link = "https://journals.aps.org/prb/recent?page=3" # 看第三页（三页基本上覆盖了当天的所有更新）
html = urlopen(start_link).read().decode('utf-8') # 打开网页
soup = BeautifulSoup(html, features='lxml') # 放入soup中
all_a_tag = soup.find_all('a', href=True) # 获取超链接标签
for a_tag in all_a_tag:
href = a_tag['href'] # 超链接字符串
if re.search('/abstract/', href): # 文章的链接
if re.search('https://journals.aps.org', href)==None: # 如果链接不是完整的，那么补充完整
href = 'https://journals.aps.org'+ href
if href not in match_href and href not in link_list and re.search('\?', href)==None: # 链接不重复
match_href.append(href)
f.write(href+'\n')
f.close()
# 获取摘要
try:
f = open('prb_all.txt', 'a', encoding='UTF-8') # 全部记录
except:
f = open('prb_all.txt', 'w', encoding='UTF-8') # 如果文件不存在
try:
f_month = open('prb_'+str(year)+'.'+str(month).rjust(2,'0')+'.txt', 'a', encoding='UTF-8') # 一个月的记录
except:
f_month = open('prb_'+str(year)+'.'+str(month).rjust(2,'0')+'.txt', 'w', encoding='UTF-8') # 如果文件不存在
f.write('\n\n['+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+'][total number='+str(len(match_href))+']\n\n\n')
f_month.write('\n\n['+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+'][total number='+str(len(match_href))+']\n\n\n')
print('total number=', len(match_href)) # 调试的时候显示这个
i00 = 0
for href in match_href:
i00 += 1
print('reading number', i00, '...') # 调试的时候显示这个
# time.sleep(random.uniform(10,110)) # 爬虫简单伪装，休息一分钟左右。如果链接个数有60个，那么程序运行时间延长60分钟。调试的时候把该语句注释。
try:
html = urlopen(href).read().decode('utf-8') # 打开文章链接
soup = BeautifulSoup(html, features='lxml') # 放入soup中
title = soup.title # 文章标题
f.write(str(title.get_text())+'\n\n')
f_month.write(str(title.get_text())+'\n\n')
f.write(str(href)+'\n\n') # 文章链接
f_month.write(str(href)+'\n\n')
abstract = re.findall('"yes"><p>.*</p><div', html, re.S)[0][9:-8] # 文章摘要
word_list = abstract.split(' ') # 划分单词
for word in word_list:
if re.search('<', word)==None and re.search('>', word)==None: # 有些内容满足过滤条件，因此信息可能会丢失。
f.write(word+' ')
f_month.write(word+' ')
f.write('\n\n\n')
f_month.write('\n\n\n')
except:
pass
f.close()

88
2021.06.07_find_common_words_in_APS_abstracts/collect_prl_abstracts.py
View File

@@ -1,88 +0,0 @@
"""
This code is supported by the website: https://www.guanjihuan.com
The newest version of this code is on the web page: https://www.guanjihuan.com/archives/13623
"""
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
from collections import Counter
import datetime
import random
import time
# time.sleep(random.uniform(0,1800)) # 爬虫简单伪装，在固定时间后0到30分钟后开始运行。调试的时候把该语句注释。
year = datetime.datetime.now().year
month = datetime.datetime.now().month
day = datetime.datetime.now().day
# 获取链接
try:
with open('prl_link_list.txt', 'r', encoding='UTF-8') as f: # 如果文件存在
link_list = f.read().split('\n') # 历史已经访问过的链接（数组类型）
except:
with open('prl_link_list.txt', 'w', encoding='UTF-8') as f: # 如果文件不存在
link_list = []
f = open('prl_link_list.txt', 'a', encoding='UTF-8') # 打开文件（补充）
f.write('\nLink list obtained on '+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+':\n')
match_href = [] # 在本次运行中满足条件的链接
for loop in range(3):
if loop == 0:
start_link = "https://journals.aps.org/prl/recent?page=1" # 看第一页
elif loop == 1:
start_link = "https://journals.aps.org/prl/recent?page=2" # 看第二页
elif loop == 2:
start_link = "https://journals.aps.org/prl/recent?page=3" # 看第三页（三页基本上覆盖了当天的所有更新）
html = urlopen(start_link).read().decode('utf-8') # 打开网页
soup = BeautifulSoup(html, features='lxml') # 放入soup中
all_a_tag = soup.find_all('a', href=True) # 获取超链接标签
for a_tag in all_a_tag:
href = a_tag['href'] # 超链接字符串
if re.search('/abstract/', href): # 文章的链接
if re.search('https://journals.aps.org', href)==None: # 如果链接不是完整的，那么补充完整
href = 'https://journals.aps.org'+ href
if href not in match_href and href not in link_list and re.search('\?', href)==None: # 链接不重复
match_href.append(href)
f.write(href+'\n')
f.close()
# 获取摘要
try:
f = open('prl_all.txt', 'a', encoding='UTF-8') # 全部记录
except:
f = open('prl_all.txt', 'w', encoding='UTF-8') # 如果文件不存在
try:
f_month = open('prl_'+str(year)+'.'+str(month).rjust(2,'0')+'.txt', 'a', encoding='UTF-8') # 一个月的记录
except:
f_month = open('prl_'+str(year)+'.'+str(month).rjust(2,'0')+'.txt', 'w', encoding='UTF-8') # 如果文件不存在
f.write('\n\n['+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+'][total number='+str(len(match_href))+']\n\n\n')
f_month.write('\n\n['+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+'][total number='+str(len(match_href))+']\n\n\n')
print('total number=', len(match_href)) # 调试的时候显示这个
i00 = 0
for href in match_href:
i00 += 1
print('reading number', i00, '...') # 调试的时候显示这个
# time.sleep(random.uniform(10,110)) # 爬虫简单伪装，休息一分钟左右。如果链接个数有60个，那么程序运行时间延长60分钟。调试的时候把该语句注释。
try:
html = urlopen(href).read().decode('utf-8') # 打开文章链接
soup = BeautifulSoup(html, features='lxml') # 放入soup中
title = soup.title # 文章标题
f.write(str(title.get_text())+'\n\n')
f_month.write(str(title.get_text())+'\n\n')
f.write(str(href)+'\n\n') # 文章链接
f_month.write(str(href)+'\n\n')
abstract = re.findall('"yes"><p>.*</p><div', html, re.S)[0][9:-8] # 文章摘要
word_list = abstract.split(' ') # 划分单词
for word in word_list:
if re.search('<', word)==None and re.search('>', word)==None: # 有些内容满足过滤条件，因此信息可能会丢失。
f.write(word+' ')
f_month.write(word+' ')
f.write('\n\n\n')
f_month.write('\n\n\n')
except:
pass
f.close()

41
2021.06.07_find_common_words_in_APS_abstracts/count_words.py
View File

@@ -1,41 +0,0 @@
"""
This code is supported by the website: https://www.guanjihuan.com
The newest version of this code is on the web page: https://www.guanjihuan.com/archives/13623
"""
import re
from collections import Counter
def main():
file_name = 'prb_all.txt'
with open(file_name, 'r', encoding='UTF-8') as f: # 打开文件
paper_list = f.read().split('\n\n\n') # 通过三个回车划分不同文章
word_list = []
ignore = ignore_words() # 过滤常见单词
for paper in paper_list:
word_list_in_one_paper = []
if len(paper)>20: # 通过字符串长度过滤日期
content_list = paper.split('\n\n') # 通过两个回车划分内容
for content in content_list:
if re.search('https://', content)==None: # 过滤文章链接
words = content.split(' ') # 通过空格划分单词
for word in words:
if word not in word_list_in_one_paper: # 一篇文章的某个单词只统计一次
if word not in ignore and len(word)>1: # 过滤词汇
word_list.append(word)
word_list_in_one_paper.append(word)
num = 300
most_common_words = Counter(word_list).most_common(num) # 统计出现最多的num个词汇
print('\n出现频率最高的前', num, '个词汇：')
for word in most_common_words:
print(word)
def ignore_words(): # 可自行增删
ignore = ['Phys.', 'the', 'to', 'of', 'in', 'under', 'and', 'by', 'The', 'at', 'with', 'up', 'be', 'above', 'below', 'are', 'is', 'for', 'that', 'as', 'we', '<a', 'abstract', 'abstract"','<span', 'which', 'We', 'such', 'has', 'two', 'these', 'it', 'all', 'results', 'result', 'each', 'have', 'between', 'on', 'an', 'can', 'also', 'from', 'Our', 'our', 'using', 'where', 'These', 'out', 'both', 'due', 'less', 'along', 'but', 'In', 'show', 'into', 'study', 'find', 'provide', 'change','not', 'open', 'this', 'show', 'into', 'study', 'find', 'provide', 'change', 'present', 'Using', 'large', 'This', 'However', 'appear', 'studied', 'obtain', 'been', 'Both', 'they', 'effects', 'effect', 'compute', 'more', 'does', 'shown', 'Based', 'reveal', 'highly', 'number', 'However,', 'was', 'near', 'full', 'based', 'several', 'suggest', 'agreement', 'predicted', 'values', 'work', 'emphasize', 'without', 'or', 'work,', 'studies', 'future', 'identify', 'present.', 'predict', 'presence', 'their', 'were', 'From', 'its', 'By', 'how', 'ground', 'observed', 'recent', 'For', 'other', 'Here', 'test', 'further', 'Its', 'similar', 'however,', 'range', 'within', 'value', 'possible', 'may', 'than', 'low', 'us', 'obtained', 'around', 'consider', 'about', 'very', 'will', 'when', 'played', 'consist', 'consists', 'Here,', 'observe', 'gives', 'It', 'over', 'cannot', 'As', 'whose', 'new', 'some', 'only', 'from', 'yields', 'shows', 'data', 'direct', 'related', 'different', 'evidence', 'role', 'function', 'origin', 'specific', 'set', 'confirm', 'give', 'Moreover', 'develop', 'including', 'could', 'used', 'means', 'allows', 'make', 'e.g.,', 'provides', 'system', 'systems', 'field', 'fields', 'model', 'model,', 'state', 'states', 'states.', 'state.', 'band', 'bands', 'method', 'methods', 'nature', 'rate', 'zero', 'single', 'theory', 'first', 'one', 'complex', 'approach', 'schemes', 'terms', 'even', 'case', 'analysis', 'weight', 'volume', 'evolution', 'well', 'external', 'measured', 'introducing', 'dependence', 'properties', 'demonstrate', 'remains', 'through', 'measurements', 'samples', 'findings', 'respect', 'investigate', 'behavior', 'importance', 'considered', 'experimental', 'increase', 'propose', 'follows', 'increase', 'emerged', 'interesting', 'behaviors', 'influenced', 'paramount', 'indicate', 'Rev.', 'concepts', 'induced', 'zone', 'regions', 'exact', 'contribution', 'behavior', 'formation', 'measurements.', 'utilizing', 'constant', 'regime', 'features', 'strength', 'compare', 'determined', 'combination', 'compare', 'determined', 'At', 'inside', 'ambient', 'then', 'important', 'report', 'Moreover,', 'Despite', 'found', 'because', 'process', 'and,', 'significantly', 'realized', 'much', 'natural', 'since', 'grows', 'any', 'compared', 'while', 'forms.', 'appears', 'indicating', 'coefficient', 'suggested', 'time', 'exhibits', 'calculations.', 'developed', 'array', 'discuss', 'field', 'becomes', 'allowing', 'indicates', 'via', 'introduce', 'considering', 'times.', 'constructed', 'explain', 'form', 'owing', 'parameters.', 'parameter', 'operation', 'probe', 'experiments', 'interest', 'strategies', 'seen', 'emerge', 'generic', 'geometry', 'numbers', 'observation', 'avenue', 'theretically', 'three', 'excellent', 'amount', 'notable', 'example', 'being', 'promising', 'latter', 'little', 'imposed', 'put', 'resource', 'together', 'produce', 'successfully','there', 'enhanced', 'this', 'great', 'dirven', 'increasing','should', 'otherwise', 'Further', 'field,', 'known', 'changes', 'still', 'beyond', 'various', 'center', 'previously', 'way', 'peculiar', 'detailed', 'understanding', 'good', 'years', 'where', 'Me', 'origins', 'years.', 'attributed', 'known,', 'them', 'reported', 'no', 'systems', 'agree', 'examined', 'rise', 'calculate', 'those', 'particular', 'relation', 'defined', 'either', 'again', 'current', 'exhibit', 'calculated', 'here', 'made', 'Further', 'consisting', 'constitutes', 'originated', 'if', 'exceed', 'access']
return ignore
if __name__ == '__main__':
main()

37
2021.11.17_zhihu/nature_physics.py
View File

@@ -1,37 +0,0 @@
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import datetime
year = datetime.datetime.now().year
month = datetime.datetime.now().month
day = datetime.datetime.now().day
f = open('nature_physics.html', 'w', encoding='UTF-8')
f.write('<meta charset="utf-8"><style type="text/css">a{text-decoration: none;color: #0a5794;}a:hover {text-decoration: underline;color: red; }</style>')
f.write('<p>'+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+' 已更新</p>')
match_href = []
start_link = "https://www.nature.com/nphys/research-articles"
html = urlopen(start_link).read().decode('utf-8') # 打开网页
soup = BeautifulSoup(html, features='lxml') # 放入soup中
all_article = soup.find_all('article', {"class":"u-full-height c-card c-card--flush"})
for article in all_article:
all_a_tag = article.find_all('a', href=True) # 获取超链接标签
for a_tag in all_a_tag:
href = a_tag['href'] # 超链接字符串
if re.search('/articles/', href): # 文章的链接
if re.search('https://www.nature.com', href)==None: # 如果链接不是完整的，那么补充完整
href = 'https://www.nature.com'+ href
if href not in match_href and re.search('\?', href)==None: # 链接不重复
match_href.append(href)
f.write('<li><a target=\"_blank\" href=\"')
f.write(href) # 文章链接
f.write('\">')
f.write(a_tag.get_text())
f.write('</a>&nbsp;&nbsp;')
time = article.find('time', {"class": "c-meta__item c-meta__item--block-at-lg"}).get_text()
f.write(time+'</li>')
f.close()

36
2021.11.17_zhihu/physics_magazine.py
View File

@@ -1,36 +0,0 @@
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import datetime
year = datetime.datetime.now().year
month = datetime.datetime.now().month
day = datetime.datetime.now().day
f = open('physics_magazine.html', 'w', encoding='UTF-8')
f.write('<meta charset="utf-8"><style type="text/css">a{text-decoration: none;color: #0a5794;}a:hover {text-decoration: underline;color: red; }</style>')
f.write('<p>'+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+' 已更新</p>')
match_href = []
start_link = "https://physics.aps.org/"
html = urlopen(start_link).read().decode('utf-8') # 打开网页
soup = BeautifulSoup(html, features='lxml') # 放入soup中
all_articles = soup.find_all('div', {"class":"feed-item-details"})
for article in all_articles:
all_a_tag = article.find_all('a', href=True) # 获取超链接标签
for a_tag in all_a_tag:
href = a_tag['href'] # 超链接字符串
if re.search('/articles/', href): # 文章的链接
if re.search('https://physics.aps.org', href)==None: # 如果链接不是完整的，那么补充完整
href = 'https://physics.aps.org'+ href
if href not in match_href:
match_href.append(href)
f.write('<li><a target=\"_blank\" href=\"')
f.write(href) # 文章链接
f.write('\">')
f.write(a_tag.get_text())
f.write('</a>&nbsp;&nbsp;')
time = article.find('time', {"class": "feed-item-date"}).get_text()
f.write(time+'</li>')
f.close()

42
2021.11.17_zhihu/prb.py
View File

@@ -1,42 +0,0 @@
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import datetime
year = datetime.datetime.now().year
month = datetime.datetime.now().month
day = datetime.datetime.now().day
f = open('prb.html', 'w', encoding='UTF-8')
f.write('<meta charset="utf-8"><style type="text/css">a{text-decoration: none;color: #0a5794;}a:hover {text-decoration: underline;color: red; }</style>')
f.write('<p>'+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+' 已更新</p>')
match_href = []
for loop in range(1):
if loop == 0:
start_link = "https://journals.aps.org/prb/recent" # 看第一页
# elif loop == 1:
# start_link = "https://journals.aps.org/prb/recent?page=2" # 看第二页
html = urlopen(start_link).read().decode('utf-8') # 打开网页
soup = BeautifulSoup(html, features='lxml') # 放入soup中
all_article = soup.find_all('div', {"class":"article panel article-result"})
for article in all_article:
all_a_tag = article.find_all('a', href=True) # 获取超链接标签
for a_tag in all_a_tag:
href = a_tag['href'] # 超链接字符串
if re.search('/abstract/', href): # 文章的链接
if re.search('https://journals.aps.org', href)==None: # 如果链接不是完整的，那么补充完整
href = 'https://journals.aps.org'+ href
if href not in match_href and re.search('\?', href)==None: # 链接不重复
match_href.append(href)
f.write('<li><a target=\"_blank\" href=\"')
f.write(href) # 文章链接
f.write('\">')
f.write(a_tag.get_text())
f.write('</a>&nbsp;&nbsp;')
info = article.find('h6', {"class": "pub-info"}).get_text()
f.write(re.findall('– Published .*', info, re.S)[0][12:]+'</li>')
f.close()

42
2021.11.17_zhihu/prl.py
View File

@@ -1,42 +0,0 @@
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import datetime
year = datetime.datetime.now().year
month = datetime.datetime.now().month
day = datetime.datetime.now().day
f = open('prl.html', 'w', encoding='UTF-8')
f.write('<meta charset="utf-8"><style type="text/css">a{text-decoration: none;color: #0a5794;}a:hover {text-decoration: underline;color: red; }</style>')
f.write('<p>'+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+' 已更新</p>')
match_href = []
for loop in range(1):
if loop == 0:
start_link = "https://journals.aps.org/prl/recent" # 看第一页
# elif loop == 1:
# start_link = "https://journals.aps.org/prl/recent?page=2" # 看第二页
html = urlopen(start_link).read().decode('utf-8') # 打开网页
soup = BeautifulSoup(html, features='lxml') # 放入soup中
all_article = soup.find_all('div', {"class":"article panel article-result"})
for article in all_article:
all_a_tag = article.find_all('a', href=True) # 获取超链接标签
for a_tag in all_a_tag:
href = a_tag['href'] # 超链接字符串
if re.search('/abstract/', href): # 文章的链接
if re.search('https://journals.aps.org', href)==None: # 如果链接不是完整的，那么补充完整
href = 'https://journals.aps.org'+ href
if href not in match_href and re.search('\?', href)==None: # 链接不重复
match_href.append(href)
f.write('<li><a target=\"_blank\" href=\"')
f.write(href) # 文章链接
f.write('\">')
f.write(a_tag.get_text())
f.write('</a>&nbsp;&nbsp;')
info = article.find('h6', {"class": "pub-info"}).get_text()
f.write(re.findall('– Published.*', info, re.S)[0][12:]+'</li>')
f.close()

66
2021.11.17_zhihu/zhihu.py
View File

@@ -1,66 +0,0 @@
"""
This code is supported by the website: https://www.guanjihuan.com
The newest version of this code is on the web page: https://www.guanjihuan.com/archives/17937
"""
from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import datetime
year = datetime.datetime.now().year
month = datetime.datetime.now().month
day = datetime.datetime.now().day
# 获取链接
# 由于没有模拟登录知乎，因此只能爬取到最新的两篇文章
authors = ["https://www.zhihu.com/people/guanjihuan/posts"] # Guan
match_href = []
for i0 in range(len(authors)):
start_link = authors[i0]
html = urlopen(start_link).read().decode('utf-8') # 打开网页
soup = BeautifulSoup(html, features='lxml') # 放入soup中
all_a_tag = soup.find_all('a', href=True) # 获取超链接标签
for a_tag in all_a_tag:
href = a_tag['href'] # 超链接字符串
if re.search('//zhuanlan.zhihu.com/p/', href) and not re.search('edit', href): # 文章的链接
if re.search('https:', href)==None: # 如果链接不是完整的，那么补充完整
href = 'https:'+ href
if href not in match_href:
match_href.append(href)
# 对链接进行排序
numbers = []
match_href_new = []
for href in match_href:
numbers.append(int(href[29:]))
numbers.sort(reverse = True)
for n in numbers:
match_href_new.append('https://zhuanlan.zhihu.com/p/'+str(n))
# 获取内容并写入文件
f = open('zhihu.html', 'w', encoding='UTF-8')
f.write('<meta charset="utf-8"><style type="text/css">a{text-decoration: none;color: #004e4e;}a:hover {text-decoration: underline;color: red; }</style>')
f.write('<p>'+str(year)+'.'+str(month).rjust(2,'0')+'.'+str(day).rjust(2,'0')+' 已更新</p>')
for href in match_href_new:
try:
html = urlopen(href).read().decode('utf-8') # 打开文章链接
soup = BeautifulSoup(html, features='lxml') # 放入soup中
title = soup.title # 文章标题
f.write('<li><a target=\"_blank\" href=\"')
f.write(str(href)) # 文章链接
f.write('\">')
f.write(str(title.get_text()[:-5]))
f.write('</a>&nbsp;&nbsp;')
author = soup.find("span", {"class": "UserLink AuthorInfo-name"})
f.write(str(author.get_text()+'&nbsp;&nbsp;'))
post_time = soup.find("div", {"class" : "ContentItem-time"})
f.write(str(post_time.get_text()[4:-6])+'</li>')
except:
pass
f.close()