Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
V
VenJob
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Mai Hoang Thai Ha
VenJob
Commits
31274088
Commit
31274088
authored
Jul 08, 2021
by
Mai Hoang Thai Ha
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fixed code style
parent
f54f8365
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
15 additions
and
20 deletions
+15
-20
lib/tasks/web_crawler.rake
+15
-20
No files found.
lib/tasks/web_crawler.rake
View file @
31274088
# Web crawler
require
'open-uri'
require
'open-uri'
require
'csv'
require
'csv'
require
'zip'
require
'zip'
namespace
:job
do
namespace
:job
do
desc
"import job"
desc
'importjob'
task
web_job_import: :environment
do
task
web_job_import: :environment
do
url
=
"https://careerbuilder.vn/vi/tim-viec-lam/nhan-vien-thiet-ke-thoi-trang.35B6D3AD.html"
url
=
'https://careerbuilder.vn/vi/tim-viec-lam/nhan-vien-thiet-ke-thoi-trang.35B6D3AD.html'
unparsed_page
=
HTTParty
.
get
(
url
)
unparsed_page
=
HTTParty
.
get
(
url
)
parsed_page
||=
Nokogiri
::
HTML
(
unparsed_page
.
body
)
parsed_page
||=
Nokogiri
::
HTML
(
unparsed_page
.
body
)
job_desc
=
parsed_page
.
css
(
'div.job-desc'
)
job_desc
=
parsed_page
.
css
(
'div.job-desc'
)
job_detail
=
parsed_page
.
css
(
'section.job-detail-content'
)
job_detail
=
parsed_page
.
css
(
'section.job-detail-content'
)
# title
# title
title
=
job_desc
.
css
(
'h1.title'
).
text
title
=
job_desc
.
css
(
'h1.title'
).
text
company
=
job_desc
.
css
(
'a.job-company-name'
).
text
company
=
job_desc
.
css
(
'a.job-company-name'
).
text
# info box
# info box
info_box
=
job_detail
.
css
(
'div.detail-box'
)
info_box
=
job_detail
.
css
(
'div.detail-box'
)
info_box_item
=
info_box
.
css
(
'ul li'
)
info_box_item
=
info_box
.
css
(
'ul li'
)
update_at
,
industry
,
type
,
salary
,
experience
,
level
,
expiration_date
=
''
update_at
,
industry
,
type
,
salary
,
experience
,
level
,
expiration_date
=
''
city
=
info_box
.
first
.
text
.
squish
.
remove
(
'Địa điểm'
).
strip
city
=
info_box
.
first
.
text
.
squish
.
remove
(
"Địa điểm"
).
strip
(
0
..
info_box_item
.
count
-
1
).
each
do
|
part
|
(
0
..
info_box_item
.
count
-
1
).
each
do
|
part
|
info
=
info_box_item
[
part
].
text
info
=
info_box_item
[
part
].
text
if
info
.
include?
(
key
=
"Ngày cập nhật"
)
if
info
.
include?
(
key
=
'Ngày cập nhật'
)
update_at
=
info
.
squish
.
remove
(
key
).
strip
update_at
=
info
.
squish
.
remove
(
key
).
strip
elsif
info
.
include?
(
key
=
"Ngành nghề"
)
elsif
info
.
include?
(
key
=
'Ngành nghề'
)
industry
=
info
.
squish
.
remove
(
key
).
strip
industry
=
info
.
squish
.
remove
(
key
).
strip
elsif
info
.
include?
(
key
=
"Hình thức"
)
elsif
info
.
include?
(
key
=
'Hình thức'
)
type
=
info
.
squish
.
remove
(
key
).
strip
type
=
info
.
squish
.
remove
(
key
).
strip
elsif
info
.
include?
(
key
=
"Lương"
)
elsif
info
.
include?
(
key
=
'Lương'
)
salary
=
info
.
squish
.
remove
(
key
).
strip
salary
=
info
.
squish
.
remove
(
key
).
strip
elsif
info
.
include?
(
key
=
"Kinh nghiệm"
)
elsif
info
.
include?
(
key
=
'Kinh nghiệm'
)
experience
=
info
.
squish
.
remove
(
key
).
strip
experience
=
info
.
squish
.
remove
(
key
).
strip
elsif
info
.
include?
(
key
=
"Cấp bậc"
)
elsif
info
.
include?
(
key
=
'Cấp bậc'
)
level
=
info
.
squish
.
remove
(
key
).
strip
level
=
info
.
squish
.
remove
(
key
).
strip
elsif
info
.
include?
(
key
=
"Hết hạn nộp"
)
elsif
info
.
include?
(
key
=
'Hết hạn nộp'
)
expiration_date
=
info
.
squish
.
remove
(
key
).
strip
expiration_date
=
info
.
squish
.
remove
(
key
).
strip
end
end
end
end
# benefit
# benefit
job_detail_row
=
job_detail
.
css
(
'div.detail-row'
)
job_detail_row
=
job_detail
.
css
(
'div.detail-row'
)
benefit_list
=
[]
benefit_list
=
[]
...
@@ -52,18 +49,16 @@ namespace :job do
...
@@ -52,18 +49,16 @@ namespace :job do
benefit
=
benefits
[
part
].
text
.
strip
benefit
=
benefits
[
part
].
text
.
strip
benefit_list
<<
benefit
benefit_list
<<
benefit
end
end
# description - requirment
# description - requirment
(
0
..
job_detail_row
.
count
-
1
).
each
do
|
part
|
(
0
..
job_detail_row
.
count
-
1
).
each
do
|
part
|
job_detail_text
=
job_detail_row
[
part
].
text
job_detail_text
=
job_detail_row
[
part
].
text
if
job_detail_text
.
include?
(
'Mô tả Công việc'
)
if
job_detail_text
.
include?
(
"Mô tả Công việc"
)
descriptions
=
job_detail_row
.
css
(
'p'
)
descriptions
=
job_detail_row
.
css
(
'p'
)
(
0
..
descriptions
.
count
-
1
).
each
do
|
desc
|
(
0
..
descriptions
.
count
-
1
).
each
do
|
desc
|
description
=
descriptions
[
desc
].
text
.
strip
description
=
descriptions
[
desc
].
text
.
strip
description_list
<<
description
description_list
<<
description
end
end
elsif
job_detail_text
.
include?
(
"Yêu Cầu Công Việc"
)
elsif
job_detail_text
.
include?
(
'Yêu Cầu Công Việc'
)
requirements
=
job_detail_row
.
css
(
'p'
)
requirements
=
job_detail_row
.
css
(
'p'
)
(
0
..
requirements
.
count
-
1
).
each
do
|
req
|
(
0
..
requirements
.
count
-
1
).
each
do
|
req
|
requirement
=
requirements
[
req
].
text
.
strip
requirement
=
requirements
[
req
].
text
.
strip
...
@@ -78,6 +73,7 @@ namespace :job do
...
@@ -78,6 +73,7 @@ namespace :job do
info
=
other_info
[
part
].
text
.
squish
.
strip
info
=
other_info
[
part
].
text
.
squish
.
strip
other_info_list
<<
info
other_info_list
<<
info
end
end
job
=
{
job
=
{
title:
title
,
title:
title
,
company:
company
,
company:
company
,
...
@@ -95,6 +91,5 @@ namespace :job do
...
@@ -95,6 +91,5 @@ namespace :job do
requirement:
requirement_list
,
requirement:
requirement_list
,
other_info:
other_info_list
other_info:
other_info_list
}
}
byebug
end
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment