Skip to content

V
VeNJob
Commit ed8bc042 by Tô Ngọc Ánh
Options

Merge branch 'crawler' into 'master' 

refactor code

See merge request !4
parents 735a713b 102be3eb
Pipeline #698 failed with stages
in 0 seconds
Showing with 57 additions and 47 deletions
Gemfile
...@@ -64,4 +64,5 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] ...@@ -64,4 +64,5 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
## ##
gem "nokogiri" gem "nokogiri"
gem 'whenever', require: false
## ##
Gemfile.lock
...@@ -64,6 +64,7 @@ GEM ...@@ -64,6 +64,7 @@ GEM
chromedriver-helper (2.1.1) chromedriver-helper (2.1.1)
archive-zip (~> 0.10) archive-zip (~> 0.10)
nokogiri (~> 1.8) nokogiri (~> 1.8)
chronic (0.10.2)
coffee-rails (4.2.2) coffee-rails (4.2.2)
coffee-script (>= 2.2.0) coffee-script (>= 2.2.0)
railties (>= 4.0.0) railties (>= 4.0.0)
...@@ -187,6 +188,8 @@ GEM ...@@ -187,6 +188,8 @@ GEM
websocket-driver (0.7.3) websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0) websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5) websocket-extensions (0.1.5)
whenever (1.0.0)
chronic (>= 0.6.3)
xpath (3.2.0) xpath (3.2.0)
nokogiri (~> 1.8) nokogiri (~> 1.8)
...@@ -214,6 +217,7 @@ DEPENDENCIES ...@@ -214,6 +217,7 @@ DEPENDENCIES
tzinfo-data tzinfo-data
uglifier (>= 1.3.0) uglifier (>= 1.3.0)
web-console (>= 3.3.0) web-console (>= 3.3.0)
whenever
RUBY VERSION RUBY VERSION
ruby 2.6.6p146 ruby 2.6.6p146
......
config/schedule.rb 0 → 100755
# Use this file to easily define all of your cron jobs.
#
# It's helpful, but not entirely necessary to understand cron before proceeding.
# http://en.wikipedia.org/wiki/Cron
# Example:
#
# set :output, "/path/to/my/cron_log.log"
#
# every 2.hours do
# command "/usr/bin/some_great_command"
# runner "MyModel.some_method"
# rake "some:great:rake:task"
# end
#
# every 4.days do
# runner "AnotherModel.prune_old_records"
# end
# Learn more: http://github.com/javan/whenever
env :PATH, ENV['PATH']
every :day, at: '12:00 pm' do
rake 'crawl:crawl_industries_locations_jobs[4]'
end
\ No newline at end of file
lib/tasks/crawler.rake
require "open-uri" require "open-uri"
@logger ||= Logger.new("#{Rails.root}/log/crawler.log")
namespace :crawl do namespace :crawl do
desc "crawl industries and locations" desc "crawl industries locations jobs"
task crawl_industries_locations: :environment do task :crawl_industries_locations_jobs, [:page, :link] => [:environment] do |task, args|
args.with_defaults(link: 'https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html')
crawl_industries_and_locations crawl_industries_and_locations
job_links = get_job_links(args[:page].to_i, args[:link])
job_links.each do |link|
next if link.empty?
crawl_job(link)
end end
desc "crawl jobs"
task crawl_jobs: :environment do
job_links = get_job_links(1)
crawl_jobs(job_links)
end end
desc "crawl all"
task all: [:crawl_industries_locations, :crawl_jobs]
end end
def get_job_links(page) def get_job_links(page, link)
job_links = [] job_links = []
page.times do |i| page.times do
document = Nokogiri::HTML(open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i}-vi.html")) document = Nokogiri::HTML(open(link))
jobs_xml = document.xpath('//div/a[@class="job_link"]/@href') jobs_xml = document.xpath('//div/a[@class="job_link"]/@href')
jobs_xml.each { |item| job_links << item.value} jobs_xml.each { |item| job_links << item.value}
next_page = document.at_css('.next-page a')
break if next_page.nil?
link = next_page[:href]
end end
job_links job_links
end end
...@@ -31,25 +32,20 @@ def crawl_company(company_link) ...@@ -31,25 +32,20 @@ def crawl_company(company_link)
uri = URI.parse(URI.escape(company_link)) #fix error: uri must be ascii only uri = URI.parse(URI.escape(company_link)) #fix error: uri must be ascii only
document = Nokogiri::HTML(open(uri)) document = Nokogiri::HTML(open(uri))
company_name = document.css(".content .name").text company_name = document.css(".content .name").text
return nil if company_name.empty? return if company_name.empty?
puts company_name puts company_name
company_address = document.css(".content p")[1].text company_address = document.css(".content p")[1].text
company_description = document.css(".main-about-us").css('.content').text company_description = document.css(".main-about-us").css('.content').text
Company.find_or_create_by(name: company_name) do |c| Company.find_or_create_by(name: company_name) do |company|
c.address = company_address company.address = company_address
c.description = company_description company.description = company_description
end end
rescue => exception rescue => exception
puts exception puts exception
return nil @logger.error "#{exception.message} - Company link: #{uri}"
end return
end
def crawl_jobs(job_links)
job_links.each do |link|
crawl_job(link)
end end
end end
...@@ -64,41 +60,24 @@ def crawl_job(job_link) ...@@ -64,41 +60,24 @@ def crawl_job(job_link)
job_company = crawl_company(job_company_link) job_company = crawl_company(job_company_link)
return if job_company.nil? return if job_company.nil?
## parse data job_location_name = document.css('.map p a').map{ |val| val.text.strip }
job_detail = document.css('.job-detail-content div.detail-box')
job_location_name = job_detail[0].css('p a').map{ |val| val.text.strip }
job_locations = Location.where(city: job_location_name) job_locations = Location.where(city: job_location_name)
job_industry_names = job_detail[1].css('ul li')[1].css('p a').map{ |val| val.text.strip } job_industry_names = document.at_xpath('//li[./strong/em[contains(@class, "mdi mdi-briefcase")]]').css('p a').map{ |val| val.text.strip }
job_industries = Industry.where(name: job_industry_names) job_industries = Industry.where(name: job_industry_names)
job_created_at = job_detail[1].css('ul li')[0].css('p').text.strip job_salary = document.at_xpath('//li[./strong/i[contains(@class, "fa fa-usd")]]/p').try(:text).try(:strip)
job_level = document.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-account")]]/p').try(:text).try(:strip)
job_salary = "" job_experience = document.at_xpath('//li[./strong/i[contains(@class, "fa fa-briefcase")]]/p').try(:text).try(:strip)
job_level = "" job_expiration_date = document.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-calendar-check")]]/p').try(:text).try(:strip)
job_experience = ""
job_expiration_date = ""
info_detail = job_detail[2].css('ul li')
info_detail.count.times do |i|
data = info_detail[i].css('p').text.strip
job_salary = data if info_detail[i].css('.fa.fa-usd').present?
job_experience = data if info_detail[i].css('.fa.fa-briefcase').present?
job_level = data if info_detail[i].css('.mdi.mdi-account').present?
job_expiration_date = data if info_detail[i].css('.mdi.mdi-calendar-check').present?
end
job_description = document.css('.job-detail-content .detail-row').to_s job_description = document.css('.job-detail-content .detail-row').to_s
##
Job.find_or_create_by(title: job_title, company_id: job_company.id) do |job| Job.find_or_create_by(title: job_title, company_id: job_company.id) do |job|
job.salary = job_salary job.salary = job_salary
job.experience = job_experience job.experience = job_experience
job.level = job_level job.level = job_level
job.expiration_date = job_expiration_date job.expiration_date = job_expiration_date
job.created_at = job_created_at
job.description = job_description job.description = job_description
job.industries << job_industries job.industries << job_industries
job.locations << job_locations job.locations << job_locations
...@@ -106,6 +85,7 @@ def crawl_job(job_link) ...@@ -106,6 +85,7 @@ def crawl_job(job_link)
puts job_title puts job_title
rescue => exception rescue => exception
puts exception puts exception
@logger.error "#{exception.message} - Job link: #{uri}"
return exception return exception
end end
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment