Merge branch 'master' of gitlab.zigexn.vn:phucth/ven-job into feature/social_authenticate

Gemfile

@@ -29,13 +29,15 @@ gem 'rsolr'
 
 
 gem 'carrierwave'
 gem 'carrierwave'
 
 
-gem 'activerecord-import'
+gem 'activerecord-import', require: false
 
 
 gem 'will_paginate'
 gem 'will_paginate'
 
 
 gem 'omniauth-facebook'
 gem 'omniauth-facebook'
 
 
 gem 'settingslogic'
 gem 'settingslogic'
+
+gem 'parallel', require: false
 # Use Active Storage variant
 # Use Active Storage variant
 # gem 'image_processing', '~> 1.2'
 # gem 'image_processing', '~> 1.2'
 
 

Gemfile.lock

@@ -178,6 +178,7 @@ GEM
       oauth2 (~> 1.1)
       oauth2 (~> 1.1)
       omniauth (~> 1.9)
       omniauth (~> 1.9)
     orm_adapter (0.5.0)
     orm_adapter (0.5.0)
+    parallel (1.19.1)
     pry (0.13.0)
     pry (0.13.0)
       coderay (~> 1.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
       method_source (~> 1.0)
@@ -324,6 +325,7 @@ DEPENDENCIES
   meta-tags
   meta-tags
   mysql2
   mysql2
   omniauth-facebook
   omniauth-facebook
+  parallel
   pry
   pry
   puma (~> 4.1)
   puma (~> 4.1)
   rails (~> 6.0.2, >= 6.0.2.2)
   rails (~> 6.0.2, >= 6.0.2.2)

app/models/city.rb

 class City < ApplicationRecord
 class City < ApplicationRecord
+  validates :title, presence: true, uniqueness: true
+  
   has_and_belongs_to_many :jobs
   has_and_belongs_to_many :jobs
 end
 end

app/models/company.rb

 class Company < ApplicationRecord
 class Company < ApplicationRecord
+  validates :title, presence: true
+  
   has_many :jobs
   has_many :jobs
 end
 end

app/models/industry.rb

 class Industry < ApplicationRecord
 class Industry < ApplicationRecord
+  validates :title, presence: true, uniqueness: true
+  
   has_and_belongs_to_many :jobs
   has_and_belongs_to_many :jobs
 end
 end

app/models/job.rb

 class Job < ApplicationRecord
 class Job < ApplicationRecord
+  scope :by_cities, -> (city_id) {includes(:cities).where("cities.id = ?", city_id).references(:cities)}
+  scope :by_industries, -> (industry_id) {includes(:industries).where("industries.id = ?", industry_id).references(:industries)}
+  scope :by_companies, -> (company_id) {where("company_id = #{company_id}")}
+
+  EXPORT_CSV_ATTRIBUTES = %w(title updated_date_job level years_of_experience salary expiration_date).freeze
+  
   belongs_to :company
   belongs_to :company
 
 
   has_many :applies
   has_many :applies
@@ -10,9 +16,15 @@ class Job < ApplicationRecord
   has_and_belongs_to_many :industries
   has_and_belongs_to_many :industries
   has_and_belongs_to_many :cities
   has_and_belongs_to_many :cities
 
 
-  scope :by_cities, -> (city_id) {includes(:cities).where("cities.id = ?", city_id).references(:cities)}
-  scope :by_industries, -> (industry_id) {includes(:industries).where("industries.id = ?", industry_id).references(:industries)}
-  scope :by_companies, -> (company_id) {where("company_id = #{company_id}")}
+  validate :updated_date_job_cannot_be_greater_than_expiration_date
+  
+  validates :title, length: { minimum: 6 }
+  validates :title, :updated_date_job, :level, :expiration_date, :salary, :min_salary, :max_salary, presence: true
+  validates :min_salary, :max_salary, numericality: { only_integer: true }
 
 
-  EXPORT_CSV_ATTRIBUTES = %w(title updated_date_job level years_of_experience salary expiration_date).freeze
+  def updated_date_job_cannot_be_greater_than_expiration_date
+    if DateTime.parse(updated_date_job).to_i > DateTime.parse(expiration_date).to_i
+      errors.add(:updated_date_job, "can't be greater than expiration date")
+    end
+  end
 end
 end

app/services/crawler_service.rb

+class CrawlerService
+  def self.convert_salary(salary)
+    return [0, 999_999_999] if salary == "Cạnh tranh"
+
+    vn_salary = salary.tr("^[0-9]{1,2}[.,]\d{1-2}", " ")
+                      .tr(",",".")
+                      .split(" ")
+                      .map { |s| (s.to_f*1_000_000).to_i }
+
+    return [0, vn_salary[0]] if salary.include? "Dưới"
+    return [vn_salary[0], 0] if salary.include? "Trên"
+
+    [vn_salary[0], vn_salary[1]]
+  end
+
+  def self.imports(job_attributes, company_attributes, cities, industries)
+    raise Exception.new "Not enough data transferred" if job_attributes.nil? || company_attributes.nil? || cities.nil? || industries.nil?
+    ActiveRecord::Base.transaction do
+      job_attributes[:company_id] = Company.find_or_create_by!(company_attributes).id
+      job = Job.find_or_create_by!(job_attributes)
+      if job.errors.full_messages.present?
+        raise Exception.new "#{job.errors.full_messages.join(",")}"
+        raise ActiveRecord::Rollback
+      end
+      cities = cities.map do |city|
+        City.find_or_create_by({title: city})
+      end
+      industries = industries.map do |industry|
+        Industry.find_or_create_by({title: industry})
+      end
+      cities.each do |city|
+        job.cities << city
+      end
+      industries.each do |industry|
+        job.industries << industry
+      end
+    end
+  end
+end
\ No newline at end of file

db/migrate/20200423044651_add_foreign_to_cities.rb

 class AddForeignToCities < ActiveRecord::Migration[6.0]
 class AddForeignToCities < ActiveRecord::Migration[6.0]
   def change
   def change
-    add_column :cities, :foreign, :boolean, :default => false
+    add_column :cities, :foreign, :boolean, default: false
   end
   end
 end
 end

db/migrate/20200511055632_add_columns_to_jobs.rb

+class AddColumnsToJobs < ActiveRecord::Migration[6.0]
+  def change
+    add_column :jobs, :min_salary, :bigint, default: 0
+    add_column :jobs, :max_salary, :bigint, default: 0
+    add_column :jobs, :benefit, :text
+    add_column :jobs, :job_requirements, :text
+    add_column :jobs, :other_information, :text
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 #
 # It's strongly recommended that you check this file into your version control system.
 # It's strongly recommended that you check this file into your version control system.
 
 
-ActiveRecord::Schema.define(version: 2020_04_23_044651) do
+ActiveRecord::Schema.define(version: 2020_05_11_055632) do
 
 
   create_table "admins", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
   create_table "admins", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.string "email", default: "", null: false
     t.string "email", default: "", null: false
@@ -89,6 +89,11 @@ ActiveRecord::Schema.define(version: 2020_04_23_044651) do
     t.bigint "company_id"
     t.bigint "company_id"
     t.datetime "created_at", precision: 6, null: false
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
+    t.bigint "min_salary", default: 0
+    t.bigint "max_salary", default: 0
+    t.text "benefit"
+    t.text "job_requirements"
+    t.text "other_information"
     t.index ["company_id"], name: "index_jobs_on_company_id"
     t.index ["company_id"], name: "index_jobs_on_company_id"
   end
   end
 
 

lib/tasks/crawler.rake

 require "nokogiri"
 require "nokogiri"
 require "open-uri"
 require "open-uri"
+require "parallel"
+require "activerecord-import"
 
 
 namespace :crawler do
 namespace :crawler do
   desc "Crawler Careerbuilder"
   desc "Crawler Careerbuilder"
-
+  
   task job: :environment do
   task job: :environment do
-    # Define exception logger
-    exception_logger = ActiveSupport::Logger.new("log/exception_logger.log")
-
-    # Define skip logger
-    skip_url_logger = ActiveSupport::Logger.new("log/skip_url_logger.log")
-
+    # Define crawler logger
+    logger = Logger.new("log/crawler_logger.log")
+    html_careerbuilder_list_jobs = Nokogiri::HTML.parse(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html"))
+    total_page = (html_careerbuilder_list_jobs.at_css(".search-result-list .container .job-found .job-found-amout p").text.tr(",việc làm","").to_i / 50.0).ceil
     # Loop page
     # Loop page
-    (10..12).each do |page|
+    (1..total_page).each do |page|
       # Fetch and parse HTML document
       # Fetch and parse HTML document
-      html_jobs = Nokogiri::HTML.parse(open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"))
-
+      html_jobs = Nokogiri::HTML.parse(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"))
       # Loop item
       # Loop item
-      html_jobs.css(".jobs-side-list .job-item").each do |item|
-        # Job attributes
-        job_attributes = {
-          title: item.css(".figure .figcaption .title a @title").text,
-          updated_date_job: item.css(".bottom-right-icon .time time").text,
-          level: nil,
-          years_of_experience: nil,
-          salary: item.css(".figure .figcaption .caption .salary").text.gsub("$ ",""),
-          expiration_date: nil,
-          job_description: nil,
-          company_id: nil,
-        }
-        # Defind cities array
-        cities = []
-
-        item.css(".figure .figcaption .caption .location ul li").each do |city|
-          city = check_exist_or_create_city(city.text.strip)
-          cities << city
-        end
-
-        if item.css(".figure .image a @href").text != "javascript:void(0);"
-          # Company attributes
-          html_company_detail = Nokogiri::HTML.parse(open(URI.encode(item.css(".figure .image a @href").text)))
-          unless html_company_detail.at_css(".jobsby-company").nil?
-            company_attributes = {
-              title: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .content .name").text,
-              address: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .content p")[1].text,
-              logo: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .img @src").text,
-              description: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .content ul").inner_html.strip
-            }
-            # Check exist or create company
-            job_attributes[:company_id] = check_exist_or_create_company(company_attributes)
+      Parallel.each(html_jobs.css(".jobs-side-list .job-item"), in_threads: 5) { |item|
+        begin
+          url = item.css(".figure .figcaption .title .job_link @href").text
+          html_job_detail = Nokogiri::HTML.parse(URI.open(URI.encode(url)))
+          if html_job_detail.at_css(".search-result-list-detail .tabs div#tab-1").nil?
+            logger.warn "Another template #{url}"
+            next
           end
           end
-        end
-        # Defind industry ids array
-        industries = []
-
-        html_job_detail = Nokogiri::HTML.parse(open(URI.encode(item.css(".figure .figcaption .title .job_link @href").text)))
-        unless html_job_detail.at_css(".search-result-list-detail").nil?
-          html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .has-background ul li").each do |ele|
-            type = ele.css("strong").text
-
+          # Set salary, min-salary, max-salary
+          if item.at_css(".figure .figcaption .caption .salary").text.include? "USD"
+            logger.warn "Another template #{url}"
+            next
+          end
+          salary = item.at_css(".figure .figcaption .caption .salary").text.gsub("$ ","")
+          min_salary, max_salary = CrawlerService.convert_salary(salary)
+          # Job attributes
+          job_attributes = {
+            title: item.at_css(".figure .figcaption .title a @title").text,
+            updated_date_job: item.at_css(".bottom-right-icon .time time").text,
+            salary: salary,
+            min_salary: min_salary,
+            max_salary: max_salary
+          }
+          html_job_detail.css(".job-detail-content .row .has-background ul li").each do |ele|
+            type = ele.at_css("strong").text
             case type
             case type
             when "Hết hạn nộp"
             when "Hết hạn nộp"
-              job_attributes[:expiration_date] = ele.css("p").text.strip
+              job_attributes[:expiration_date] = ele.at_css("p").text.squish
             when "Cấp bậc"
             when "Cấp bậc"
-              job_attributes[:level] = ele.css("p").text.strip
+              job_attributes[:level] = ele.at_css("p").text.squish
             when "Kinh nghiệm"
             when "Kinh nghiệm"
-              job_attributes[:years_of_experience] = ele.css("p").text.strip
+              job_attributes[:years_of_experience] = ele.at_css("p").text.squish
             end
             end
           end
           end
-          html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-box .industry p a").each do |ele|
-            industry = check_exist_or_create_industry(ele.text.gsub(",","").strip)
-            industries << industry
-          end
-          # Get description for job attributes
-          description = ""
-
           html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-row").each do |ele|
           html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-row").each do |ele|
-            description << ele.inner_html
-          end
-          # Set description for job attributes
-          job_attributes[:job_description] = description.strip
-        else
-          skip_url_logger.info "another template #{item.css(".figure .figcaption .title .job_link @href").text}"
-        end
-        # Create job
-        job = check_exist_or_create_job(job_attributes)
-        # Create city_job
-        if cities.count > 0
-          cities.each do |city|
-            job.cities << city
+            next if ele.at_css(".detail-title").nil?
+            type = ele.at_css(".detail-title").text
+            case type
+            when "Phúc lợi "
+              job_attributes[:benefit] = ele.at_css("ul").inner_html.squish
+            when "Mô tả Công việc"
+              job_attributes[:job_description] = ele.inner_html.squish.gsub("<h3 class=\"detail-title\">Mô tả Công việc</h3>","")
+            when "Yêu Cầu Công Việc"
+              job_attributes[:job_requirements] = ele.inner_html.squish.gsub("<h3 class=\"detail-title\">Yêu Cầu Công Việc</h3>","")
+            when "Thông tin khác"
+              job_attributes[:other_information] = ele.inner_html.squish.gsub("<h3 class=\"detail-title\">Thông tin khác</h3>","")
+            end
           end
           end
-        end
-        # Create industry_job
-        if industries.count > 0
-          industries.each do |industry|
-            job.industries << industry
+          next if item.at_css(".figure .image a @href").text == "javascript:void(0);"
+          # Company attributes
+          html_company_detail = Nokogiri::HTML.parse(URI.open(URI.encode(item.css(".figure .image a @href").text)))
+          next if html_company_detail.at_css(".jobsby-company").nil?
+          company_css = ".jobsby-company .company-introduction .company-info .info "
+          company_attributes = {
+            title: html_company_detail.at_css(company_css + ".content .name").text,
+            address: html_company_detail.css(company_css + ".content p")[1].text,
+            logo: html_company_detail.at_css(company_css + ".img @src").text,
+            description: html_company_detail.at_css(company_css + ".content ul").inner_html.squish
+          }
+          # Defind cities array
+          cities = item.css(".figure .figcaption .caption .location ul li").map do |city|
+            city.text.squish
           end
           end
-        end
-        rescue
-          exception_logger.info "Error url: #{item.css(".figure .figcaption .title .job_link @href").text}"
+          # Defind industries array
+          industries = html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-box .industry p a").map do |industry|
+            industry.text.tr(",","").squish
+          end              
+
+          sleep rand
+          Mutex.new.synchronize {
+            result = CrawlerService.imports(job_attributes, company_attributes, cities, industries)
+          }
+          sleep rand
+
+          logger.info "Crawl success url : #{url}"
+        rescue => e
+          logger.error e
           next
           next
-      end
+        end
+      }
     end
     end
   end
   end
+
   task city: :environment do
   task city: :environment do
     # Fetch and parse HTML document
     # Fetch and parse HTML document
-    html_cities = Nokogiri::HTML.parse(open("https://careerbuilder.vn/tim-viec-lam.html"))
-    unless html_cities.at_css(".find-jobsby-categories .main-jobs-by-location").nil?
-      # Defind cities array
-      cities = []
-      # Get city in country
-      html_cities.css(".find-jobsby-categories .main-jobs-by-location .jobs-in-country .list-jobs-by-country li a").each do |title|
-        city = {
-          title: title.text.gsub("Việc làm tại ","").strip,
-          foreign: false
-        }
-        cities << city
-      end
-      # Get city foreign
-      html_cities.css(".find-jobsby-categories .main-jobs-by-location .overseas-jobs .list-overseas-jobs li a").each do |title|
-        city = {
-          title: title.text.strip,
-          foreign: true
-        }
-        cities << city
-      end
-      if cities.count > 0
-        City.import cities
-      end
+    html_cities = Nokogiri::HTML.parse(URI.open("https://careerbuilder.vn/tim-viec-lam.html"))
+    # Get city in country
+    cities_in_country = html_cities.css(".find-jobsby-categories .main-jobs-by-location .jobs-in-country .list-jobs-by-country li a").map do |title|
+      {
+        title: title.text.gsub("Việc làm tại ","").squish,
+        foreign: false
+      }
     end
     end
-  end
-  task industry: :environment do
-    # Fetch and parse HTML document
-    html_industries = Nokogiri::HTML.parse(open("https://careerbuilder.vn/tim-viec-lam.html"))
-    unless html_industries.at_css(".find-jobsby-categories .list-of-working-positions").nil?
-      # Defind industries array
-      industries = []
-      # Get industry
-      html_industries.css(".find-jobsby-categories .list-of-working-positions .list-jobs li a").each do |title|
-        industry = {
-          title: title.text.strip
-        }
-        industries << industry
-      end
-      if industries.count > 0
-        Industry.import industries
-      end
+    # Get city foreign
+    cities_foreign = html_cities.css(".find-jobsby-categories .main-jobs-by-location .overseas-jobs .list-overseas-jobs li a").map do |title|
+      {
+        title: title.text.squish,
+        foreign: true
+      }
     end
     end
-  end
-  def check_exist_or_create_company(company_attributes)
-    find_company = Company.find_or_create_by(company_attributes)
-    return find_company.id
-  end
-
-  def check_exist_or_create_industry(industry_title)
-    industries = Industry.where("title LIKE ?", industry_title)
-    if industries.count == 0
-      industry = Industry.create(title: industry_title)
-    else
-      industry = industries[0]
+    cities = cities_in_country + cities_foreign
+    if cities.length > 0
+      City.import cities
     end
     end
-    return industry
   end
   end
 
 
-  def check_exist_or_create_city(city_title)
-    cities = City.where("title LIKE ?", city_title)
-    if cities.count == 0
-      city = City.create(title: city_title)
-    else
-      city = cities[0]
+  task industry: :environment do
+    # Fetch and parse HTML document
+    html_industries = Nokogiri::HTML.parse(URI.open("https://careerbuilder.vn/tim-viec-lam.html"))
+    # Get industry
+    industries = html_industries.css(".find-jobsby-categories .list-of-working-positions .list-jobs li a").map do |title|
+      {
+        title: title.text.squish
+      }
+    end
+    if industries.length > 0
+      Industry.import industries
     end
     end
-    return city
-  end
-
-  def check_exist_or_create_job(job_attributes)
-    job = Job.find_or_create_by(job_attributes)
-    return job
   end
   end
 end
 end