Skip to content

V
venjob_nth
Commit 37488f20 by Ngô Trung Hưng
Options

autoload

parent bf921c4c
Pipeline #788 canceled with stages
in 0 seconds
Showing with 286 additions and 268 deletions
config/application.rb
...@@ -10,10 +10,7 @@ module Venjob ...@@ -10,10 +10,7 @@ module Venjob
class Application < Rails::Application class Application < Rails::Application
# Initialize configuration defaults for originally generated Rails version. # Initialize configuration defaults for originally generated Rails version.
config.load_defaults 5.2 config.load_defaults 5.2
config.autoload_paths += [ config.autoload_paths << Rails.root.join('lib')
Rails.root.join('lib/src'),
Rails.root.join('lib/src/base'),
Rails.root.join('lib/src/interface')]
# Settings in config/environments/* take precedence over those specified here. # Settings in config/environments/* take precedence over those specified here.
# Application configuration can go into files in config/initializers # Application configuration can go into files in config/initializers
# -- all .rb files in that directory are automatically loaded after loading # -- all .rb files in that directory are automatically loaded after loading
......
lib/src/base/base.rb
...@@ -5,83 +5,87 @@ require 'open-uri' ...@@ -5,83 +5,87 @@ require 'open-uri'
require 'logger' require 'logger'
# Crawler data # Crawler data
class Base module Src
attr_accessor :job, :page module Base
class Base
def initialize(page) attr_accessor :job, :page
@job = {}
@page = page def initialize(page)
end @job = {}
@page = page
def logger end
@logger ||= Logger.new(Rails.root.join('log', 'crawl.log'))
end def logger
@logger ||= Logger.new(Rails.root.join('log', 'crawl.log'))
def create_data end
take_data
job def create_data
rescue StandardError => e take_data
logger.error "Crawler data job have error: #{e}" job
end rescue StandardError => e
logger.error "Crawler data job have error: #{e}"
private end
def take_data private
job[:name] = fill_name
job[:company_name] = fill_company_name def take_data
job[:city_name] = fill_city_name job[:name] = fill_name
job[:created_date] = fill_created_date job[:company_name] = fill_company_name
job[:expiration_date] = fill_expiration_date job[:city_name] = fill_city_name
job[:salary] = fill_salary job[:created_date] = fill_created_date
job[:industry_name] = fill_industry_name job[:expiration_date] = fill_expiration_date
job[:description] = fill_description job[:salary] = fill_salary
job[:level] = fill_lever job[:industry_name] = fill_industry_name
job[:exprience] = fill_experience job[:description] = fill_description
end job[:level] = fill_lever
job[:exprience] = fill_experience
def fill_name end
page.search('.apply-now-content .job-desc .title').text
end def fill_name
page.search('.apply-now-content .job-desc .title').text
def fill_company_name end
page.search('.apply-now-content .job-desc .job-company-name').text
end def fill_company_name
page.search('.apply-now-content .job-desc .job-company-name').text
def fill_city_name end
page.search('.detail-box .map p a').map(&:text).join(',')
end def fill_city_name
page.search('.detail-box .map p a').map(&:text).join(',')
def fill_created_date end
page.search('.item-blue .detail-box:nth-child(1) ul li:nth-child(1) p')[0].try(:text)
end def fill_created_date
page.search('.item-blue .detail-box:nth-child(1) ul li:nth-child(1) p')[0].try(:text)
def fill_expiration_date end
page.xpath('//ul//li[last()]//p').last.text
end def fill_expiration_date
page.xpath('//ul//li[last()]//p').last.text
def fill_salary end
page.xpath('//ul//li[position()=1]//p')[1].text
end def fill_salary
page.xpath('//ul//li[position()=1]//p')[1].text
def fill_industry_name end
industries = page.xpath('//ul//li[position()=2]//p//a').map(&:text)
industries.map(&:strip).join(',') def fill_industry_name
end industries = page.xpath('//ul//li[position()=2]//p//a').map(&:text)
industries.map(&:strip).join(',')
def fill_description end
job[:description] = page.search('.tabs .tab-content .detail-row').to_s
end def fill_description
job[:description] = page.search('.tabs .tab-content .detail-row').to_s
def exist_experience? end
noname = page.search('//ul//li').text
noname.include?('Kinh nghiệm') def exist_experience?
end noname = page.search('//ul//li').text
noname.include?('Kinh nghiệm')
def fill_lever end
exist_experience? ? page.xpath('//ul//li[position()=3]//p')[1].text.strip : page.xpath('//ul//li[position()=2]//p')[1].text
end def fill_lever
exist_experience? ? page.xpath('//ul//li[position()=3]//p')[1].text.strip : page.xpath('//ul//li[position()=2]//p')[1].text
def fill_experience end
exist_experience? ? page.xpath('//ul//li[position()=2]//p')[1].text.strip : ''
def fill_experience
exist_experience? ? page.xpath('//ul//li[position()=2]//p')[1].text.strip : ''
end
end
end end
end end
lib/src/crawler.rb
...@@ -3,63 +3,65 @@ ...@@ -3,63 +3,65 @@
require 'open-uri' require 'open-uri'
# Crawler data # Crawler data
class Crawler module Src
attr_accessor :number_link class Crawler
attr_accessor :number_link
def initialize(number_link) def initialize(number_link)
@number_link = number_link @number_link = number_link
end end
def path_to_first_link def path_to_first_link
Rails.root.join('tmp', 'link.txt') Rails.root.join('tmp', 'link.txt')
end end
def logger def logger
@logger ||= Logger.new(Rails.root.join('log', 'crawler.log')) @logger ||= Logger.new(Rails.root.join('log', 'crawler.log'))
end end
def link_make_stop_crawler def link_make_stop_crawler
file = File.readlines(path_to_first_link, 'r') if File.exist?(path_to_first_link) file = File.readlines(path_to_first_link, 'r') if File.exist?(path_to_first_link)
file.blank? ? 'NOT' : file.join file.blank? ? 'NOT' : file.join
end end
def safe_link(url) def safe_link(url)
Nokogiri::HTML(URI.open(URI.escape(url))) Nokogiri::HTML(URI.open(URI.escape(url)))
end end
def crawl_link def crawl_link
website_companies = [] website_companies = []
number_link.times do |i| number_link.times do |i|
page = Nokogiri::HTML(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i + 1}-vi.html")) page = Nokogiri::HTML(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i + 1}-vi.html"))
link_companies = page.search('.figcaption .caption @href') link_companies = page.search('.figcaption .caption @href')
website_companies += link_companies.map(&:value).uniq website_companies += link_companies.map(&:value).uniq
link_jobs = page.search('.figcaption .title .job_link @href').text link_jobs = page.search('.figcaption .title .job_link @href').text
break if link_jobs.include?(link_make_stop_crawler) break if link_jobs.include?(link_make_stop_crawler)
end
website_companies.select(&:present?)
rescue StandardError => e
logger.error "Crawler link on page have error #{e}"
end end
website_companies.select(&:present?)
rescue StandardError => e
logger.error "Crawler link on page have error #{e}"
end
def craw_data_cities def craw_data_cities
page = Nokogiri::HTML(URI.open('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html')) page = Nokogiri::HTML(URI.open('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html'))
locations = page.search('#location option').map(&:text) locations = page.search('#location option').map(&:text)
locations.each_with_index do |val, index| locations.each_with_index do |val, index|
area = index > City::RANGE ? City.areas['international'] : City.areas['domestic'] area = index > City::RANGE ? City.areas['international'] : City.areas['domestic']
City.find_or_create_by(name: val) { |city| city.area = area } City.find_or_create_by(name: val) { |city| city.area = area }
end
end end
end
def craw_data_companies def craw_data_companies
crawl_link.each do |url| crawl_link.each do |url|
page = safe_link(url) page = safe_link(url)
company_name = page.search('.company-info .content .name').text company_name = page.search('.company-info .content .name').text
Company.find_or_create_by(name: company_name) do |company| Company.find_or_create_by(name: company_name) do |company|
company.address = page.search('.company-info .info .content p:nth-child(3)').text company.address = page.search('.company-info .info .content p:nth-child(3)').text
company.short_description = page.search('.main-about-us .content').text company.short_description = page.search('.main-about-us .content').text
end
rescue StandardError => e
logger.error "Crawler data companies has error: #{e}"
end end
rescue StandardError => e
logger.error "Crawler data companies has error: #{e}"
end end
end end
end end
lib/src/crawler_job.rb
# frozen_string_literal: true # frozen_string_literal: true
# Crawler data job # Autoload
class CrawlerJob < Crawler module Src
SIZE_LI = 8 # Crawler job
class CrawlerJob < Crawler
SIZE_LI = 8
def crawl_link def crawl_link
website_jobs = [] website_jobs = []
number_link.times do |i| number_link.times do |i|
page = Nokogiri::HTML(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i + 1}-vi.html")) page = Nokogiri::HTML(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i + 1}-vi.html"))
link_jobs = page.search('.figcaption .title .job_link @href') link_jobs = page.search('.figcaption .title .job_link @href')
link_jobs.each do |val| link_jobs.each do |val|
link = val.value link = val.value
return website_jobs if link.include?(link_make_stop_crawler) return website_jobs if link.include?(link_make_stop_crawler)
website_jobs << link website_jobs << link
end
end end
website_jobs
rescue StandardError => e
logger.error "Crawler link jobs on page have error #{e}"
end end
website_jobs
rescue StandardError => e
logger.error "Crawler link jobs on page have error #{e}"
end
def parse_data def parse_data
@parse_data ||= crawl_link.reverse! @parse_data ||= crawl_link.reverse!
end end
def refresh_first_link def refresh_first_link
File.write(path_to_first_link, parse_data.last) File.write(path_to_first_link, parse_data.last)
end end
def craw_data_jobs def craw_data_jobs
parse_data.each do |path| parse_data.each do |path|
page = safe_link(path) page = safe_link(path)
if page.search('.item-blue .detail-box:nth-child(1) ul li:nth-child(1) p')[0].present? if page.search('.item-blue .detail-box:nth-child(1) ul li:nth-child(1) p')[0].present?
@data = RedInterface.new(page).create_data @data = Src::Interface::RedInterface.new(page).create_data
elsif page.search('section .template-200').text.present? elsif page.search('section .template-200').text.present?
@data = BlueInterface.new(page).create_data @data = Src::Interface::BlueInterface.new(page).create_data
elsif page.search('.DetailJobNew ul li').size == SIZE_LI && page.search('.right-col ul li').text.exclude?('Độ tuổi') elsif page.search('.DetailJobNew ul li').size == SIZE_LI && page.search('.right-col ul li').text.exclude?('Độ tuổi')
@data = GreenInterface.new(page).create_data @data = Src::Interface::GreenInterface.new(page).create_data
end
add_data(@data)
refresh_first_link
end end
add_data(@data)
refresh_first_link
end end
end
def add_data(data) def add_data(data)
id_company = (Company.find_by name: data[:company_name]).try(:id) || Company::COMPANY_SECURITY id_company = (Company.find_by name: data[:company_name]).try(:id) || Company::COMPANY_SECURITY
job = Job.create(name: data[:name], job = Job.create(name: data[:name],
company_id: id_company, company_id: id_company,
level: data[:level], level: data[:level],
experience: data[:exprience], experience: data[:exprience],
salary: data[:salary], salary: data[:salary],
create_date: data[:created_date], create_date: data[:created_date],
expiration_date: data[:expiration_date], expiration_date: data[:expiration_date],
description: data[:description]) description: data[:description])
create_industry_relation(data[:industry_name], job) create_industry_relation(data[:industry_name], job)
create_city_relation(data[:city_name], job) create_city_relation(data[:city_name], job)
rescue StandardError => e rescue StandardError => e
logger.error "Crawler data jobs has error: #{e}" logger.error "Crawler data jobs has error: #{e}"
end end
def create_industry_relation(data, job) def create_industry_relation(data, job)
industries = data.split(',') industries = data.split(',')
industries.each do |val| industries.each do |val|
val.gsub!('&amp;', '&') if val.include?('&amp;') val.gsub!('&amp;', '&') if val.include?('&amp;')
industry = Industry.find_or_create_by name: val.strip industry = Industry.find_or_create_by name: val.strip
job.industries << industry job.industries << industry
end
end end
end
def create_city_relation(data, job) def create_city_relation(data, job)
cities = data.split(',') cities = data.split(',')
cities.each do |city| cities.each do |city|
city = City.find_or_create_by(name: city.strip, area: City.areas['domestic']) city = City.find_or_create_by(name: city.strip, area: City.areas['domestic'])
job.cities << city job.cities << city
end
end end
end end
end end
lib/src/interface/blue_interface.rb
# frozen_string_literal: true # frozen_string_literal: true
# Inherience from base # Inherience from base
class BlueInterface < Base module Src
def fill_company_name module Interface
page.search('.top-job .top-job-info .tit_company').text class BlueInterface < Base::Base
end def fill_company_name
page.search('.top-job .top-job-info .tit_company').text
def fill_city_name end
page.search('.info-workplace .value a').map(&:text).join(',')
end def fill_city_name
page.search('.info-workplace .value a').map(&:text).join(',')
def fill_expiration_date end
page.xpath('//ul//li[position()=4]//div').text
end def fill_expiration_date
page.xpath('//ul//li[position()=4]//div').text
def fill_salary end
page.xpath('//ul//li[position()=3]//div').text
end def fill_salary
page.xpath('//ul//li[position()=3]//div').text
def fill_industry_name end
page.xpath('//ul//li[position()=5]//div').text
end def fill_industry_name
page.xpath('//ul//li[position()=5]//div').text
def fill_description end
page.search('.left-col').to_s
end def fill_description
page.search('.left-col').to_s
def exist_level? end
noname = page.xpath('//ul//li[position()=2]/b').last.text
noname.include?('Cấp bậc') def exist_level?
end noname = page.xpath('//ul//li[position()=2]/b').last.text
noname.include?('Cấp bậc')
def fill_lever end
exist_level? ? page.xpath('//ul//li[position()=2]/div').last.text : ''
end def fill_lever
exist_level? ? page.xpath('//ul//li[position()=2]/div').last.text : ''
def fill_experience end
page.xpath('//ul//li[position()=7]/b').text
def fill_experience
page.xpath('//ul//li[position()=7]/b').text
end
end
end end
end end
lib/src/interface/green_interface.rb
# frozen_string_literal: true # frozen_string_literal: true
# ahihi # ahihi
class GreenInterface < Base module Src
def fill_name module Interface
page.search('.info-company h1').text class GreenInterface < Base::Base
end def fill_name
page.search('.info-company h1').text
def fill_company_name end
page.search('.info-company .text-job h2').text
end def fill_company_name
page.search('.info-company .text-job h2').text
def fill_city_name end
page.search('.DetailJobNew ul li:nth-child(1) a').text
end def fill_city_name
page.search('.DetailJobNew ul li:nth-child(1) a').text
def fill_expiration_date end
page.xpath('//ul//li[last()-1]//span').children[1].text
end def fill_expiration_date
page.xpath('//ul//li[last()-1]//span').children[1].text
def fill_salary end
page.xpath('//ul//li[last()-2]//span').text
end def fill_salary
page.xpath('//ul//li[last()-2]//span').text
def fill_industry_name end
page.search('.DetailJobNew li:nth-child(3) span').text.strip
end def fill_industry_name
page.search('.DetailJobNew li:nth-child(3) span').text.strip
def fill_description end
page.search('.left-col .detail-row').text
end def fill_description
page.search('.left-col .detail-row').text
def fill_lever end
page.search('.DetailJobNew li:nth-child(2) span').text.strip
end def fill_lever
page.search('.DetailJobNew li:nth-child(2) span').text.strip
def exist_experience? end
noname = page.search('.DetailJobNew li span').text
noname.include?('Kinh nghiệm') def exist_experience?
end noname = page.search('.DetailJobNew li span').text
noname.include?('Kinh nghiệm')
def fill_experience end
exist_experience? ? page.search('.DetailJobNew li:nth-child(5) span').text.strip : ''
def fill_experience
exist_experience? ? page.search('.DetailJobNew li:nth-child(5) span').text.strip : ''
end
end
end end
end end
lib/src/interface/red_interface.rb
# frozen_string_literal: true # frozen_string_literal: true
# Inherience from base # Inherience from base
class RedInterface < Base module Src
module Interface
class RedInterface < Base::Base
end
end
end end
lib/tasks/crawler.rake
...@@ -10,9 +10,9 @@ namespace :crawler do ...@@ -10,9 +10,9 @@ namespace :crawler do
company.address = 'Vui lòng xem trong mô tả công việc' company.address = 'Vui lòng xem trong mô tả công việc'
company.short_description = 'Vui lòng xem trong mô tả công việc' company.short_description = 'Vui lòng xem trong mô tả công việc'
end end
cw = Crawler.new(NUMBER_LINK_WILL_BE_CRAWLER) cw = Src::Crawler.new(NUMBER_LINK_WILL_BE_CRAWLER)
cw.craw_data_cities cw.craw_data_cities
cw.craw_data_companies cw.craw_data_companies
CrawlerJob.new(NUMBER_LINK_WILL_BE_CRAWLER).craw_data_jobs Src::CrawlerJob.new(NUMBER_LINK_WILL_BE_CRAWLER).craw_data_jobs
end end
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment