Tô Ngọc Ánh · @anhtn · Thanh Hung Pham · Tô Ngọc Ánh · Thanh Hung Pham · Tô Ngọc Ánh
--- a/config/database.yml
+++ b/config/database.yml
 #
 default: &default
  adapter: mysql2
-  encoding: utf8
+  encoding: utf8mb4
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
+  uri = URI.parse(CGI.escape(job_link)) # fix error: uri must be ascii only
+  document = Nokogiri::HTML(URI.open(uri))
+  job_title = document.at_css('.job-desc p.title').text
+  return if job_title.empty?
+
+  job_company_link = document.at_css('.job-desc a.job-company-name')[:href]
+  job_company = crawl_company(job_company_link)
+  return if job_company.nil?
+
+  job_location_name = document.css('.map p a').map { |val| val.text.strip }
+  job_locations = Location.where(city: job_location_name)
+
+  job_industry_names = document.at_xpath('//li[./strong/em[contains(@class, "mdi mdi-briefcase")]]').css('p a').map { |val| val.text.strip }
+  job_industries = Industry.where(name: job_industry_names)
+
+  job_salary = document.at_xpath('//li[./strong/i[contains(@class, "fa fa-usd")]]/p').try(:text).try(:strip)
--- a/Gemfile
+++ b/Gemfile
  gem 'listen', '>= 3.0.5', '< 3.2'
  # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
  gem 'spring'
-  gem 'spring-watcher-listen', '~> 2.0.0'
  gem 'dotenv-rails'
+  gem 'spring-watcher-listen', '~> 2.0.0'
--- a/lib/tasks/crawler.rake
+++ b/lib/tasks/crawler.rake
-require "open-uri"
+require 'open-uri'
+
 @logger ||= Logger.new("#{Rails.root}/log/crawler.log")

 namespace :crawl do
-  desc "crawl industries locations jobs"
-  task :crawl_industries_locations_jobs, [:page, :link] => [:environment] do |task, args|
+  desc 'crawl industries locations jobs'
+  task :crawl_industries_locations_jobs, %i[page link] => [:environment] do |_, args|
--- a/lib/tasks/ftp_import.rake 0 → 100644
+++ b/lib/tasks/ftp_import.rake 0 → 100644
+require 'csv'
+require 'zip'
+require_relative '../common/ftp'
+
+namespace :ftp_import do
--- a/lib/tasks/ftp_import.rake 0 → 100644
+++ b/lib/tasks/ftp_import.rake 0 → 100644
+  Zip::File.open(file) do |zip_file|
+    zip_file.each do |f|
+      fpath = File.join(destination, f.name)
+      zip_file.extract(f, fpath) unless File.exist?(fpath)
+    end
+  end
+end
+
+def import_job(direction)
+  # i = 0
+  CSV.foreach("#{direction}/jobs.csv", headers: true) do |row|
+    # i+=1
+    next if row['name'].blank? || !row['category'].is_a?(String) || row['company name'].blank?
+
+    title = row['name'].strip
+    company = Company.find_or_create_by(name: row['company name']) do |c|
--- a/lib/tasks/ftp_import.rake 0 → 100644
+++ b/lib/tasks/ftp_import.rake 0 → 100644
+    locations = Location.where(city: locations_name)
+    locations = locations_name.map { |city| Location.create(oversea: false, city: city) } if locations.empty?
+    description = "Benefits:\n#{row['benefit']}\n"\
+                  "Descriptions:\n#{row['description']}\n"\
+                  "Requirements:\n#{row['requirement']}"
+
+    Job.find_or_create_by(title: title, company_id: company.id, level: level, salary: salary) do |job|
+      job.industries << industry
+      job.locations << locations
+      job.description = description
+    end
+    puts title
+  end
+rescue StandardError => e
+  puts e
+  @logger.error e.message
--- a/lib/tasks/crawler.rake → lib/common/crawler.rb
+++ b/lib/tasks/crawler.rake → lib/common/crawler.rb
-@logger ||= Logger.new("#{Rails.root}/log/crawler.log")
-
-namespace :crawl do
-  desc "crawl industries locations jobs"
-  task :crawl_industries_locations_jobs, [:page, :link] => [:environment] do |task, args|
-    args.with_defaults(link: 'https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html')
-    crawl_industries_and_locations
-    job_links = get_job_links(args[:page].to_i, args[:link])
+require 'open-uri'
+
+class Crawler
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def crawl_data(page, base_link)
--- a/lib/common/csv.rb 0 → 100644
+++ b/lib/common/csv.rb 0 → 100644
+require 'csv'
+require './lib/common/extract_zip'
+
+class CsvImport
+  include ExtractZip
+
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def import_job(direction)
+    index = 0
+    CSV.foreach("#{direction}/jobs.csv", headers: true) do |row|
--- a/lib/common/csv.rb 0 → 100644
+++ b/lib/common/csv.rb 0 → 100644
+require 'csv'
+require './lib/common/extract_zip'
+
+class CsvImport
+  include ExtractZip
+
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def import_job(direction)
+    index = 0
+    CSV.foreach("#{direction}/jobs.csv", headers: true) do |row|
+      index += 1
+      next if integer?(row['category'])
--- a/lib/common/csv.rb 0 → 100644
+++ b/lib/common/csv.rb 0 → 100644
+
+      Job.find_or_create_by(title: title, company_id: company.id, level: level, salary: salary) do |job|
+        job.industries << industry
+        job.locations << locations
+        job.description = description
+      end
+      puts title
+    end
+  rescue StandardError => e
+    puts e
+    @logger.error "Job #{index}: #{e.message}"
+  end
+
+  private
+
+  def integer?(str)