Skip to content

V
venjob
Commit 941b0f25 by Huỳnh Thiên Phước
Options

Fix code crawler

parent a3d60343
Pipeline #714 canceled with stages
in 0 seconds
Showing with 27 additions and 21 deletions
lib/src/crawler.rb
...@@ -32,8 +32,8 @@ ...@@ -32,8 +32,8 @@
link.gsub!('\u2019',"'") link.gsub!('\u2019',"'")
end end
next if link == 'javascript:void(0);' next if link == 'javascript:void(0);'
elsif link != 'https://careerbuilder.vn/vi/nha-tuyen-dung/hr-vietnam\xE2\x80\x99s-ess-client.35A4EFBA.html' if link != 'https://careerbuilder.vn/vi/nha-tuyen-dung/hr-vietnam\xE2\x80\x99s-ess-client.35A4EFBA.html'
company_page = Nokogiri::HTML(URI.open(URI.parse(CGI.escape(link)))) company_page = Nokogiri::HTML(URI.open(URI.parse(URI.escape(link))))
if !(company_page.search('p.name').text).nil? if !(company_page.search('p.name').text).nil?
begin begin
name_company = company_page.search('p.name').text name_company = company_page.search('p.name').text
...@@ -61,8 +61,8 @@ ...@@ -61,8 +61,8 @@
if link.include?('\u2013') if link.include?('\u2013')
link.gsub!('\u2013','–') link.gsub!('\u2013','–')
end end
page_job = Nokogiri::HTML(URI.open(URI.parse(CGI.escape(link)))) page_job = Nokogiri::HTML(URI.open(URI.parse(URI.escape(link))))
get_row = page_job.search('div.bg-blue div.row') get_row = page_job.search('div.bg-blue div.row')
if get_row != "" if get_row != ""
get_name_company = page_job.search('div.job-desc a.job-company-name').text.strip get_name_company = page_job.search('div.job-desc a.job-company-name').text.strip
company_table = Company.find_by(name: "#{get_name_company}") company_table = Company.find_by(name: "#{get_name_company}")
...@@ -101,23 +101,29 @@ ...@@ -101,23 +101,29 @@
company_id: company_table.id) company_id: company_table.id)
end end
end end
job_table = Job.find_by(title: title_job) if !company_table.nil?
if !job_table.nil? job_table = Job.find_by(title: title_job)
location_rel = get_row.css('div.map p a').children.map{ |location| location.text.strip } if !job_table.nil?
location_rel.each do |loc| location_rel = get_row.css('div.map p a').children.map{ |location| location.text.strip }
city_table = City.find_by(name: "#{loc}") location_rel.each do |loc|
puts "Created City: #{job_table.id} - #{city_table.id}.#{loc}" city_table = City.find_by(name: "#{loc}")
city_jobs = CityJob.create!(job_id: job_table.id, city_id: city_table.id) if CityJob.find_by(job_id: job_table.id, city_id: city_table.id) == nil
end puts "Created City: #{job_table.id} - #{city_table.id}.#{loc}"
industry_rel = get_row.css('li a').children.map{ |industry| industry.text.strip } city_jobs = CityJob.create!(job_id: job_table.id, city_id: city_table.id)
industry_rel.each do |ind| end
industry_table = Industry.find_by(name: "#{ind}") end
puts "Created Industry: #{job_table.id} - #{industry_table.id}.#{ind}" industry_rel = get_row.css('li a').children.map{ |industry| industry.text.strip }
industry_jobs = IndustryJob.create!(job_id: job_table.id, industry_id: industry_table.id) industry_rel.each do |ind|
industry_table = Industry.find_by(name: "#{ind}")
if IndustryJob.find_by(job_id: job_table.id, industry_id: industry_table.id) == nil
puts "Created Industry: #{job_table.id} - #{industry_table.id}.#{ind}"
industry_jobs = IndustryJob.create!(job_id: job_table.id, industry_id: industry_table.id)
end
end
end
end end
end end
end end
end
end end
end end
end end
......
lib/src/crontab.rb
...@@ -7,8 +7,8 @@ class Crontab ...@@ -7,8 +7,8 @@ class Crontab
link.gsub!('\u2019',"'") link.gsub!('\u2019',"'")
end end
next if link == 'javascript:void(0);' next if link == 'javascript:void(0);'
elsif link != 'https://careerbuilder.vn/vi/nha-tuyen-dung/hr-vietnam\xE2\x80\x99s-ess-client.35A4EFBA.html' if link != 'https://careerbuilder.vn/vi/nha-tuyen-dung/hr-vietnam\xE2\x80\x99s-ess-client.35A4EFBA.html'
company_page = Nokogiri::HTML(URI.open(URI.parse(CGI.escape(link)))) company_page = Nokogiri::HTML(URI.open(URI.parse(URI.escape(link))))
if !(company_page.search('p.name').text).nil? if !(company_page.search('p.name').text).nil?
begin begin
name_company = company_page.search('p.name').text name_company = company_page.search('p.name').text
...@@ -34,7 +34,7 @@ class Crontab ...@@ -34,7 +34,7 @@ class Crontab
if link.include?('\u2013') if link.include?('\u2013')
link.gsub!('\u2013','–') link.gsub!('\u2013','–')
end end
page_job = Nokogiri::HTML(URI.open(URI.parse(CGI.escape(link)))) page_job = Nokogiri::HTML(URI.open(URI.parse(URI.escape(link))))
get_row = page_job.search('div.bg-blue div.row') get_row = page_job.search('div.bg-blue div.row')
if get_row != "" if get_row != ""
get_name_company = page_job.search('div.job-desc a.job-company-name').text.strip get_name_company = page_job.search('div.job-desc a.job-company-name').text.strip
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment