Trịnh Hoàng Phúc · @phucth · phuctmZigexn · Trịnh Hoàng Phúc · phuctmZigexn · Trịnh Hoàng Phúc
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
+    exception_logger = Logger.new("log/exception_logger.log")

    # Define skip logger
-    skip_url_logger = ActiveSupport::Logger.new("log/skip_url_logger.log")
+    skip_url_logger = Logger.new("log/skip_url_logger.log")

    # Loop page
-    (10..12).each do |page|
+    (1..2).each do |page|
      # Fetch and parse HTML document
-      html_jobs = Nokogiri::HTML.parse(open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"))
-
+      html_jobs = Nokogiri::HTML.parse(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"))
      # Loop item
      html_jobs.css(".jobs-side-list .job-item").each do |item|
+        # Set salary, min-salary, max-salary
--- a/db/migrate/20200511055632_add_columns_to_jobs.rb 0 → 100644
+++ b/db/migrate/20200511055632_add_columns_to_jobs.rb 0 → 100644
+class AddColumnsToJobs < ActiveRecord::Migration[6.0]
+  def change
+    add_column :jobs, :min_salary, :bigint, :default => 0
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
    # Define skip logger
-    skip_url_logger = ActiveSupport::Logger.new("log/skip_url_logger.log")
+    skip_url_logger = Logger.new("log/skip_url_logger.log")

    # Loop page
-    (10..12).each do |page|
+    (1..2).each do |page|
      # Fetch and parse HTML document
-      html_jobs = Nokogiri::HTML.parse(open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"))
-
+      html_jobs = Nokogiri::HTML.parse(URI.open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{page}-vi.html"))
      # Loop item
      html_jobs.css(".jobs-side-list .job-item").each do |item|
+        # Set salary, min-salary, max-salary
+        salary = item.at_css(".figure .figcaption .caption .salary").text.gsub("$ ","")
+        if salary == "Cạnh tranh"
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake

 namespace :crawler do
  desc "Crawler Careerbuilder"
-
+  
  task job: :environment do
    # Define exception logger
-    exception_logger = ActiveSupport::Logger.new("log/exception_logger.log")
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
-              title: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .content .name").text,
-              address: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .content p")[1].text,
-              logo: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .img @src").text,
-              description: html_company_detail.css(".jobsby-company .company-introduction .company-info .info .content ul").inner_html.strip
-            }
-            # Check exist or create company
-            job_attributes[:company_id] = check_exist_or_create_company(company_attributes)
-          end
-        end
        # Defind industry ids array
        industries = []

-        html_job_detail = Nokogiri::HTML.parse(open(URI.encode(item.css(".figure .figcaption .title .job_link @href").text)))
-        unless html_job_detail.at_css(".search-result-list-detail").nil?
+        html_job_detail = Nokogiri::HTML.parse(URI.open(URI.encode(item.css(".figure .figcaption .title .job_link @href").text)))
+        if html_job_detail.at_css(".search-result-list-detail .container .no-gutters").present?
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
          end
-          # Get description for job attributes
-          description = ""

-          html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-row").each do |ele|
-            description << ele.inner_html
+          if industries.length > 0
+            industries.each do |industry|
+              job.industries << industry
+            end
          end
-          # Set description for job attributes
-          job_attributes[:job_description] = description.strip
        else
-          skip_url_logger.info "another template #{item.css(".figure .figcaption .title .job_link @href").text}"
+          skip_url_logger.info "another template #{item.at_css(".figure .figcaption .title .job_link @href").text}"
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
-        job = check_exist_or_create_job(job_attributes)
-        # Create city_job
-        if cities.count > 0
-          cities.each do |city|
-            job.cities << city
-          end
-        end
-        # Create industry_job
-        if industries.count > 0
-          industries.each do |industry|
-            job.industries << industry
-          end
-        end
-        rescue
-          exception_logger.info "Error url: #{item.css(".figure .figcaption .title .job_link @href").text}"
+        rescue Exception => e
--- a/app/models/job.rb
+++ b/app/models/job.rb
 class Job < ApplicationRecord
+  validates :title, presence: true
--- a/app/models/city.rb
+++ b/app/models/city.rb
 class City < ApplicationRecord
+  validates :title, presence: true
--- a/app/models/company.rb
+++ b/app/models/company.rb
 class Company < ApplicationRecord
+  validates :title, presence: true
--- a/app/models/industry.rb
+++ b/app/models/industry.rb
 class Industry < ApplicationRecord
+  validates :title, presence: true
--- a/app/services/city_service.rb 0 → 100644
+++ b/app/services/city_service.rb 0 → 100644
+class CityService
+  def import cities
+    City.import cities
+  end
+
+  def check_exist_or_create_city city_title
+    cities = City.where("title LIKE ?", city_title)
--- a/app/models/job.rb
+++ b/app/models/job.rb
  has_and_belongs_to_many :industries
  has_and_belongs_to_many :cities

-  scope :by_cities, -> (city_id) {includes(:cities).where("cities.id = ?", city_id).references(:cities)}
-  scope :by_industries, -> (industry_id) {includes(:industries).where("industries.id = ?", industry_id).references(:industries)}
-  scope :by_companies, -> (company_id) {where("company_id = #{company_id}")}
+  validate :updated_date_job_cannot_be_greater_than_expiration_date, on: :create
+  
+  validates :title, length: { minimum: 6 }
+  validates :title, :updated_date_job, :level, :expiration_date, :salary, :min_salary, :max_salary, presence: true, on: :create
--- a/app/models/job.rb
+++ b/app/models/job.rb
  has_and_belongs_to_many :industries
  has_and_belongs_to_many :cities

-  scope :by_cities, -> (city_id) {includes(:cities).where("cities.id = ?", city_id).references(:cities)}
-  scope :by_industries, -> (industry_id) {includes(:industries).where("industries.id = ?", industry_id).references(:industries)}
-  scope :by_companies, -> (company_id) {where("company_id = #{company_id}")}
+  validate :updated_date_job_cannot_be_greater_than_expiration_date, on: :create
--- a/app/services/crawler_service.rb 0 → 100644
+++ b/app/services/crawler_service.rb 0 → 100644
+class CrawlerService
+  def self.convert_salary salary
+    if salary == "Cạnh tranh"
+      [0, 999999999]
+    elsif salary.include? "Dưới"
+      max_salary = (salary.gsub("Dưới ","").gsub(" Tr VND","").gsub(",",".").to_f*1000000).to_i
--- a/app/services/crawler_service.rb 0 → 100644
+++ b/app/services/crawler_service.rb 0 → 100644
+      [0, 999999999]
+    elsif salary.include? "Dưới"
+      max_salary = (salary.gsub("Dưới ","").gsub(" Tr VND","").gsub(",",".").to_f*1000000).to_i
+      [0, max_salary]
+    elsif salary.include? "Trên"
+      min_salary = (salary.gsub("Trên ","").gsub(" Tr VND","").gsub(",",".").to_f*1000000).to_i
+      max_salary = 999999999
+      [min_salary, max_salary]
+    else
+      range_salary = salary.split("-")
+      min_salary = (range_salary[0].gsub("$ ","").gsub(" Tr ","").to_f*1000000).to_i
+      max_salary = (range_salary[1].gsub(" Tr VND","").gsub(" ","").to_f*1000000).to_i
+      [min_salary, max_salary]
+    end
+  end
+end
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
-          cities << city
+        html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .has-background ul li").each do |ele|
+          type = ele.at_css("strong").text
+          case type
+          when "Hết hạn nộp"
+            job_attributes[:expiration_date] = ele.at_css("p").text.squish
+          when "Cấp bậc"
+            job_attributes[:level] = ele.at_css("p").text.squish
+          when "Kinh nghiệm"
+            job_attributes[:years_of_experience] = ele.at_css("p").text.squish
+          end
        end
-
-        if item.css(".figure .image a @href").text != "javascript:void(0);"
+        html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-row").each do |ele|
+          if ele.at_css("h3").present?
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
          end
-          # Set description for job attributes
-          job_attributes[:job_description] = description.strip
-        else
-          skip_url_logger.info "another template #{item.css(".figure .figcaption .title .job_link @href").text}"
        end
        # Create job
-        job = check_exist_or_create_job(job_attributes)
+        job = JobService.check_exist_or_create_job job_attributes
+        if job.errors.full_messages.present?
+          logger.error "#{job.errors.full_messages.join(",")}"
+          next
+        end
+        # Defind cities array
+        cities = item.css(".figure .figcaption .caption .location ul li").map do |city|
+          city = City.find_or_create_by({title: city.text.squish}) 
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
-          expiration_date: nil,
-          job_description: nil,
-          company_id: nil,
+          title: item.at_css(".figure .figcaption .title a @title").text,
+          updated_date_job: item.at_css(".bottom-right-icon .time time").text,
+          salary: salary,
+          min_salary: min_salary,
+          max_salary: max_salary
        }
-        # Defind cities array
-        cities = []
-
-        item.css(".figure .figcaption .caption .location ul li").each do |city|
-          city = check_exist_or_create_city(city.text.strip)
-          cities << city
+        html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .has-background ul li").each do |ele|
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
+          next
+        end
+        # Defind cities array
+        cities = item.css(".figure .figcaption .caption .location ul li").map do |city|
+          city = City.find_or_create_by({title: city.text.squish}) 
+        end
        # Create city_job
-        if cities.count > 0
+        if cities.length > 0
          cities.each do |city|
            job.cities << city
          end
        end
+        # Defind industries array
+        industries = html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-box .industry p a").map do |ele|
+          industry = Industry.find_or_create_by({title: ele.text.gsub(",","").squish})
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
+        html_job_detail.css(".search-result-list-detail .tabs #tab-1 .job-detail-content .detail-row").each do |ele|
+          if ele.at_css("h3").present?
+            type = ele.at_css("h3").text
+            case type
+            when "Phúc lợi "
+              job_attributes[:benefit] = ele.at_css("ul").inner_html.squish
+            when "Mô tả Công việc"
+              job_attributes[:job_description] = ele.inner_html.squish.gsub("<h3 class=\"detail-title\">Mô tả Công việc</h3>","")
+            when "Yêu Cầu Công Việc"
+              job_attributes[:job_requirements] = ele.inner_html.squish.gsub("<h3 class=\"detail-title\">Yêu Cầu Công Việc</h3>","")
+            when "Thông tin khác"
+              job_attributes[:other_information] = ele.inner_html.squish.gsub("<h3 class=\"detail-title\">Thông tin khác</h3>","")
+            end
+          end
+        end
+        if item.at_css(".figure .image a @href").text != "javascript:void(0);"