-
Notifications
You must be signed in to change notification settings - Fork 0
/
bdjobs.py
72 lines (61 loc) · 2.31 KB
/
bdjobs.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
import urllib.request
from bs4 import BeautifulSoup
import re
import csv
from datetime import date,timedelta
# We parse the data from the bdjobs.com home page
dt= date.today()
r = urllib.request.urlopen('http://bdjobs.com/').read()
soup = BeautifulSoup(r, "lxml")
#%%
#ind_jobs finds the tags for industrial categories
#fun_jobs finds the tags for the functional categories
ind_jobs = soup.find_all("div", class_="category-list padding-mobile industrial")
fun_jobs = soup.find_all("div", class_="category-list padding-mobile functional active")
#Break the text for both job categories into lines
jobs_f = fun_jobs[0].text.split('\n')
jobs_i = ind_jobs[0].text.split('\n')
#%%
job_class = []
'''For each line in functional job list, if line is not empty
add new job to job list'''
for row in jobs_f:
if(len(row)>0 and not('More' in row or 'Less' in row)):
n = re.findall('\d+', row )
job_class.append(['Functional',row.split('(')[0],int(n[0])])
'''For each line in industrial job list, if line is not empty
add new job to job list'''
for row in jobs_i:
if(len(row)>0 and not('More' in row or 'Less' in row)):
n = re.findall('\d+', row )
job_class.append(['Industrial',row.split('(')[0],int(n[0])])
#write file with relevant info about categorical jobs
with open(dt.strftime('BdJobs %Y%m%d.csv'), 'w', newline='') as f:
writer = csv.writer(f)
writer.writerows(job_class)
#%%
#Get all the hot jobs in the bdjobs homepage
hot = soup.find_all("div", class_="hotJobsCompany")
HotJobs = []
for tag in hot:
#find job title for the hot job
titles = [m.start() for m in re.finditer('title', str(tag))]
ll = len(str(tag))
names = []
for st in titles:
cnt = 0
#the following part gets the job name from the title tag
for j in range(st,ll):
if(str(tag)[j] == '"'):
cnt += 1
if(cnt==1):
tt1 = j
elif(cnt==2):
tt2 = j
break
names.append(str(tag)[tt1+1:tt2])
HotJobs.append(names)
#write hot jobs to file
with open(dt.strftime('HotJobs %Y%m%d.csv'), 'w', newline='',encoding="utf-16") as f:
writer = csv.writer(f)
writer.writerows(HotJobs)