forked from realslimkarthik/local-python-docs
-
Notifications
You must be signed in to change notification settings - Fork 0
/
crawl.py
178 lines (159 loc) · 5.46 KB
/
crawl.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
import time
import os
from urllib.parse import urlparse,urljoin
import requests
from requests import Session
from bs4 import BeautifulSoup as bs
def check_errors(request):
def inner_func(cls, method, url, **kwargs):
request_success = False
while not request_success:
try:
start = time.time()
print(f'Trying to {method} data from {url}...')
response = request(cls, method, url, **kwargs)
time_elapsed = time.time() - start
print(
f'The response from the server after {time_elapsed:.2f} seconds '\
f'was {len(response.text)} characters big. URL: {response.url}'
)
response.raise_for_status()
except (
requests.exceptions.ConnectionError, requests.exceptions.SSLError
):
print('Something went wrong. Retrying...')
print(
f'Server URL: {url}), failed while trying to connect.'
)
except requests.exceptions.HTTPError:
raw_response = response.content[:20] if len(response.content) > 50 else response.content
try:
payload = kwargs['data']
except KeyError:
payload = 'none'
error_message = f'Server URL: {response.url}, ' \
f'failed with status code ({response.status_code}). ' \
f'Raw response: {raw_response}' \
f'Request payload: {payload}'
print(error_message)
if response.status_code < 499:
# return if client error
request_success = True
else:
request_success = True
time.sleep(0.2)
return response
return inner_func
class Session(Session):
"""
Overrriden version of requests.Session that checks for errors
after completing the request.
"""
@check_errors
def request(self, method, url, **kwargs):
return super().request(method, url, **kwargs)
session = Session()
def load_links(path):
with open(path) as file:
data = file.read()
return data.split("\n")
def dump_link(url, path):
with open(path, 'a') as file:
file.write(url + "\n")
def get_file(url, remote_file):
file_name = url.split('/')[-1]
o = urlparse(url)
temp = o.path
temp = location + temp
ind = temp.find(file_name)
temp1 = temp[:ind]
if not os.path.exists(temp1):
os.makedirs(temp1)
ext = open(temp, 'w')
ext.write(remote_file)
ext.close()
def link_collect(url):
if url in visited:
link.remove(url)
return -1
response = session.get(url)
html_doc = response.text
get_file(url, html_doc)
visited.add(url)
dump_link(url, path="visited.txt")
#If link returns an HTML page then parse using BeautifulSoup
if url.find('.html') + 1:
soup = bs(html_doc)
else:
return
hRefs = set(soup.find_all('a'))
for hRef in hRefs:
try:
#If link doesn't have href attribute then continue
hRef['href']
except KeyError:
continue
else:
#If link obtained is an inline link then ignore
if hRef['href'].find('#') + 1:
continue
elif hRef['href'].find('.html') + 1:
temp = hRef.get('href')
#Used to get details about the link such as the host name, url, etc.
o = urlparse(urljoin(url,temp))
if o.netloc == 'docs.python.org':
u = o.geturl()
if u not in visited:
if u not in link:
print(f"Added {u} to the queue")
link.add(u)
dump_link(u, "dumps.txt")
else:
print(f"Collected duplicate- {u}")
link_collect(u)
ext_files = soup.find_all('link') + soup.find_all('script')
for j in set(ext_files):
try:
temp = j['src']
except KeyError:
try:
temp = j['href']
except KeyError:
continue
temp1 = urljoin(url, temp)
if temp1 in visited:
continue
if temp.startswith("_static") and temp not in link:
link_collect(temp1)
print(f"Collected static {temp1}")
continue
print(f"Added {temp1} to the queue")
if temp1 not in link:
link.add(temp1)
dump_link(temp1, "dumps.txt")
else:
# it's already there
# collect the duplicate
link_collect(temp1)
print(f"Collected duplicate {temp1}")
if __name__ == "__main__":
location = '/var/www/pydocs/'
link = set()
_link = load_links("dumps.txt")
if any(_link):
link.update(_link)
else:
link.add('http://docs.python.org/tutorial/datastructures.html')
visited = set()
already_visited = load_links(path="visited.txt")
if any(already_visited):
visited.update(already_visited)
while True:
time.sleep(0.2)
try:
l = next(iter(link))
except IndexError:
break
i = link_collect(l)
if i != -1:
print(f"{len(visited)} pages been mirrored already.")