fix part 2

814a4af9 · Ngô Trung Hưng · 5683fa11 · 814a4af9 · 814a4af9
Commit 814a4af9 authored Jul 28, 2020 by Ngô Trung Hưng
Show whitespace changes
Inline Side-by-side

Showing with 52 additions and 27 deletions

lib/src/interface_web.rb
+43 -24

lib/tasks/crawler.rake
+9 -3

No files found.
--- a/lib/src/interface_web.rb
+++ b/lib/src/interface_web.rb
@@ -4,11 +4,13 @@ require 'open-uri'
 # Crawler data
 class InterfaceWeb
-  INTERNATION = 0
+  COMPANY_SECURITY = 1
-  VIETNAM = 1
+  SIZE_LI_INTERFACE_5 = 10
+  INTERNATIONAL = 0
+  DOMESTIC = 1
  RANGE = 69
-  def self.crawl_link(page)
+  def crawl_link(page)
    puts "Crawling link on page...\nPLease wait...\n"
    data = []
    website_companies = []
@@ -31,15 +33,16 @@ class InterfaceWeb
    data << website_companies << website_jobs
  end
-  def self.link_job_and_companies
-    @link_job_and_companies ||= crawl_link(3)
+  def link_job_and_companies
+    @link_job_and_companies ||= crawl_link(2)
  end
  def self.safe_link(url)
    Nokogiri::HTML(URI.parse(URI.escape(url)))
  end
-  def self.craw_data_cities
+  def craw_data_cities
    page = Nokogiri::HTML(URI.open('https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html'))
    puts "Crawling data location... \n. \n. \n."
    data_list_cities = []
@@ -50,7 +53,7 @@ class InterfaceWeb
    end
    puts 'Save data to database...'
    data_list_cities.each_with_index do |val, index|
-      area = index > RANGE ? INTERNATION : VIETNAM
+      area = index > RANGE ? INTERNATIONAL : DOMESTIC
      City.find_or_create_by(name: val) do |city|
        city.name = val
        city.area = area
@@ -58,7 +61,7 @@ class InterfaceWeb
    end
  end
-  def self.craw_data_companies
+  def craw_data_companies
    puts 'Crawl data companies'
    link_crawl = link_job_and_companies
    link_crawl[0].each do |url|
@@ -91,10 +94,11 @@ class InterfaceWeb
    end
  end
-  def self.add_data(name, company_name, city_name, created_date, expiration_date, salary, industry_name, description, level, exprience)
+  private
-    begin
+  def add_data(name, company_name, city_name, created_date, expiration_date, salary, industry_name, description, level, exprience)
    id_company = Company.find_by name: company_name
-      id_company = id_company.present? ? id_company.id : 1
+    id_company = id_company.present? ? id_company.id : COMPANY_SECURITY
    id_job = Job.create!(name: name,
                        company_id: id_company,
                        level: level,
@@ -108,9 +112,10 @@ class InterfaceWeb
    rescue StandardError => e
    puts e
  end
-  end
-  def self.crawl_data_jobs_interface_1(page)
+  private
+  def crawl_data_jobs_interface_1(page)
    name = page.search('.apply-now-content .job-desc .title').text
    company_name = page.search('.apply-now-content .job-desc .job-company-name').text
    location = []
@@ -141,7 +146,9 @@ class InterfaceWeb
    add_data(name, company_name, city_name, created_date, expiration_date, salary, industry_name, description, level, exprience)
  end
-  def self.crawl_data_jobs_interface_2(page)
+  private
+  def crawl_data_jobs_interface_2(page)
    name = page.search('.apply-now-content .job-desc .title').text
    company_name = page.search('.top-job .top-job-info .tit_company').text
    locations = []
@@ -163,7 +170,9 @@ class InterfaceWeb
    add_data(name, company_name, city_name, created_date, expiration_date, salary, industry_name, description, level, exprience)
  end
-  def self.crawl_data_jobs_interface_5(page)
+  private
+  def crawl_data_jobs_interface_5(page)
    name = page.search('.info-company h1').text
    company_name = page.search('.info-company .text-job h2').text
    city_name = page.search('.DetailJobNew ul li:nth-child(1) a').text
@@ -177,26 +186,36 @@ class InterfaceWeb
    add_data(name, company_name, city_name, created_date, expiration_date, salary, industry_name, description, level, exprience)
  end
-  def self.make_foreign_industries_table(data, id_job)
+  private
+  def make_foreign_industries_table(data, id_job)
+    unless data.blank? && id_job.blank?       
      content = data.split(',')
      content.each do |val|
        val.gsub!('&amp;', '&') if val.include?('&amp;')
-      id_industry = Industry.find_by name: val.strip
+        data_industry = Industry.find_by name: val.strip
-      id_industry = id_industry.blank? ? Industry.create!(name: val.strip).id : id_industry.id
+        id_industry = data_industry.blank? ? Industry.create!(name: val.strip).id : data_industry.id
        IndustryJob.create!(industry_id: id_industry, job_id: id_job)
      end
    end
+  end
-  def self.make_foreign_cities_table(data, id_job)
+  private
+  def make_foreign_cities_table(data, id_job)
+    unless data.blank? && id_job.blank?      
      cities = data.split(',')
      cities.each do |city|
-      id_cities = City.find_by name: city.strip
+        data_city = City.find_by name: city.strip
-      id_cities = id_cities.blank? ? City.create!(name: city.strip, area: 1).id : id_cities.id
+        id_cities = data_city.blank? ? City.create!(name: city.strip, area: DOMESTIC).id : data_city.id
        CityJob.create!(job_id: id_job, city_id: id_cities)
      end
    end
+  end
+  public
-  def self.make_data
+  def make_data
    puts 'Please wait for crawl jobs data! . . .'
    link_crawl = link_job_and_companies
    arr_link = []
@@ -206,11 +225,11 @@ class InterfaceWeb
    end
    arr_link.reverse!.each_with_index do |path, i|
      page = Nokogiri::HTML(URI.open(URI.parse(URI.escape(path))))
-      if !page.search('.item-blue .detail-box:nth-child(1) ul li:nth-child(1) p')[0].nil?
+      if page.search('.item-blue .detail-box:nth-child(1) ul li:nth-child(1) p')[0].present?
        crawl_data_jobs_interface_1(page)
      elsif page.search('section .template-200').text.present?
        crawl_data_jobs_interface_2(page)
-      elsif page.search('.DetailJobNew ul li').size == 10 && !page.search('.right-col ul li').text.include?('Độ tuổi')
+      elsif page.search('.DetailJobNew ul li').size == SIZE_LI_INTERFACE_5 && !page.search('.right-col ul li').text.include?('Độ tuổi')
        crawl_data_jobs_interface_5(page)
      end
      puts "#{i} - #{path}"

--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
@@ -6,8 +6,14 @@ require 'src/interface_web'
 # rake task
 namespace :crawler do
  task populate: :environment do
-    InterfaceWeb.craw_data_cities
+    Company.find_or_create_by(name: 'Bảo mật') do |company|
-    InterfaceWeb.craw_data_companies
+      company.name = 'Bảo mật'
-    InterfaceWeb.make_data
+      company.address = 'Vui lòng xem trong mô tả công việc'
+      company.short_description = 'Vui lòng xem trong mô tả công việc'
+    end
+    cw = InterfaceWeb.new
+    cw.craw_data_cities
+    cw.craw_data_companies
+    cw.make_data
  end
 end