-
Notifications
You must be signed in to change notification settings - Fork 0
/
origin.py
92 lines (73 loc) · 2.88 KB
/
origin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
import requests
import io
import os
import sys
import time
ua = UserAgent()
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding = 'utf-8')
public_headers = {
'user-agent' : ua.random,
"Accept-Encoding": "gzip, deflate, br",
}
def url_get(url):
request = requests.get(url, headers = public_headers)
request.encoding = request.apparent_encoding
request = request.text
request.encode('utf-8')
print(request)
soup = BeautifulSoup(request, 'html.parser')
try:
with open("source.txt", "w", encoding = "utf-8") as txt_file:
txt_file.write(soup.prettify())
print("網頁內容已成功寫入網頁內容.txt")
except Exception as e:
print("寫入檔案時出現錯誤:", e)
def get_title_and_chatper():
with open('source.txt', 'r', encoding = 'utf-8') as file:
html_content = file.read()
soup = BeautifulSoup(html_content, 'html.parser')
title = soup.find('title').text.strip()
title = title.split("最")[0].strip()
# print(title)
if os.path.exists(title):
print(f'{title} - 資料夾已經存在。')
else:
os.makedirs(title)
chatpers = soup.select('.catalog li a')
for chapter in chatpers:
if chapter == "#":
continue
chapter_url = chapter['href']
chapter_title = chapter.text.strip()
# print(f'{chapter_title}')
chapter_file = f"{title}/{chapter_title}.txt"
with open(chapter_file, 'w', encoding = 'utf-8') as chapter_file:
# 下載章節,慢速
try:
chapter_requset_demo = requests.get(chapter_url,
headers = public_headers,
timeout = 5)
if chapter_requset_demo.status_code == 200:
chapter_requset_demo.encoding = chapter_requset_demo.apparent_encoding
chapter_text = chapter_requset_demo.text
chapter_file.write(chapter_text)
time.sleep(1)
else:
print('出狀況咧')
print(f'{chapter_title} - 已經寫入')
except requests.ConnectionError as rc:
print("連線出問題 - ", str(rc))
except requests.Timeout as rt:
print('請求超時 - ', str(rt))
except requests.RequestException as rr:
print('請求錯誤 - ', str(rr))
except Exception as e:
print('其他錯誤 - ', str(e))
print(f"已建立{len(chatpers)}個章節檔案。")
if __name__ == '__main__':
book_num = int(input())
url = f'https://www.69shuba.com/book/{book_num}/'
# url_get(url = url)
get_title_and_chatper()