-
Notifications
You must be signed in to change notification settings - Fork 1
/
main.py
140 lines (111 loc) · 4.94 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
"""
This file contains the basic code to download and store all the data for the 5th and 8th class exam from the Punjab Education Commission
Website. This site achieves that by doing the following:
1. It runs through the entire list of possible roll numbers from '00-000-000' to '99-999-999'.
2. After every page load, it checks if a result was returned. If it was, it stores that result into a local sqplite db indexed
by the roll number, otherwise it commits a NULL to it.
3. After the run is complete, it takes the db and processes the data.
"""
import logging
import random
import time
from collections import namedtuple
from optparse import OptionParser
from os import path
from pony import orm
from requests import post
# Random wait time
distribution = list(range(1000, 5001, 500))
wait = random.choice(distribution)
# Setting up the option parser
parser = OptionParser()
parser.add_option("-s", "--start-num", dest="start", type="int", default=0)
parser.add_option("-e", "--end-num", dest="end", type="int", default=10000000)
(options, args) = parser.parse_args()
# Setting up logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d-%Y %H:%M:%S,',
filename=path.join("logs", "data-{:0>8}-{:0>8}.log".format(options.start, options.end)),
filemode='a'
)
logger = logging.getLogger(__name__)
# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logger.addHandler(console)
# Named tuple for RollNo
RollNo = namedtuple("RollNo", ['roll_no1', 'roll_no2', 'roll_no3', "search"])
# Database Setup
DBNAME = "data-{:0>8}-{:0>8}.sqlite".format(options.start, options.end)
DIRNAME = "data"
db = orm.Database()
db.bind('sqlite', path.join(DIRNAME, DBNAME), create_db=True)
class Record(db.Entity):
idx = orm.Required(int)
rollno1 = orm.Required(str)
rollno2 = orm.Required(str)
rollno3 = orm.Required(str)
html = orm.Required(str)
error = orm.Required(bool)
db.generate_mapping(create_tables=True)
@orm.db_session
def visit(url, rollno, invalid, idx):
"""
This method visits the given site, fills the form, checks if a valid result is generated, adds the valid result
and the valid roll number to the valid dict and valid list respectively.
:param url:
:param rollno:
:param invalid:
"""
try:
res = post(url, rollno._asdict())
if res.text.find(invalid) != -1:
Record(rollno1=rollno[0], rollno2=rollno[1], rollno3=rollno[2], html="NULL", error=True, idx=idx)
else:
Record(rollno1=rollno[0], rollno2=rollno[1], rollno3=rollno[2], html=res.text, error=False, idx=idx)
except Exception as e:
logger.error(str(e))
Record(rollno1=rollno[0], rollno2=rollno[1], rollno3=rollno[2], html="NULL", error=True, idx=idx)
def download_data(bounds):
(from_num, to_num) = bounds
rn1 = [str(x).zfill(2) for x in range(0, 100)]
rn2 = [str(x).zfill(3) for x in range(0, 1000)]
rn3 = [str(x).zfill(3) for x in range(0, 1000)]
logger.info("Generating the list of Roll Numbers")
RNLIST = [RollNo(a, b, c, "") for a in rn1 for b in rn2 for c in rn3]
URL = "http://pec.edu.pk"
INVALID_RESULT = "No Result found"
logger.info("Start Parameter is: {}".format(from_num))
logger.info("End Parameter is: {}".format(to_num))
logger.info("Saving data to database {}".format(DBNAME))
try:
with orm.db_session:
last_record = orm.max(r.id for r in Record)
if last_record == to_num:
logger.info("No new data to download. Exiting...")
return
if last_record:
last_idx = orm.select(r.idx for r in Record if r.id == last_record)[:][0]
start = last_idx + 1
else:
start = from_num
logger.info("Starting the Brute Force Search from position {}".format(start))
logger.info("Process started at {}".format(time.strftime('%c')))
for idx, rn in enumerate(RNLIST[start:to_num]):
if idx % wait == 0: time.sleep(5)
if idx % 10 == 0:
logger.info("Downloading data for Roll No. {}".format("-".join(rn[:3])))
logger.info("Downloading Roll No. {}".format("-".join(rn[:3])))
visit(URL, rn, invalid=INVALID_RESULT, idx=RNLIST.index(rn))
logger.info("Process ended at {}".format(time.strftime('%c')))
except KeyboardInterrupt as e:
logger.error(str(e))
logger.error("Recieved Keyboard Interrupt. Exiting.")
if __name__ == '__main__':
download_data((options.start, options.end))