Merge branch 'hungpt_import_csv' into 'master'

Hungpt import csv

See merge request !2

Gemfile

@@ -51,6 +51,7 @@ group :development do
   gem 'spring-watcher-listen', '~> 2.0.0'
   gem 'better_errors'
   gem 'binding_of_caller'
+  gem 'rubyzip'
 end
 
 # Windows does not include zoneinfo files, so bundle the tzinfo-data gem

Gemfile.lock

@@ -194,6 +194,7 @@ DEPENDENCIES
   mysql2 (>= 0.3.18, < 0.5)
   puma (~> 3.7)
   rails (~> 5.1.1)
+  rubyzip
   sass-rails (~> 5.0)
   selenium-webdriver
   spring

config/schedule.rb

@@ -24,4 +24,5 @@ set :output, { error: 'log/cron_error_log.log', standard: 'log/cron_log.log' }
 
 every 1.day, at: '12:00 pm' do
   rake 'crawler:load'
+  rake 'import:csv'
 end

db/migrate/20170628014416_create_job_types.rb

 class CreateJobTypes < ActiveRecord::Migration[5.1]
   def change
     create_table :job_types do |t|
-      t.string :name
+      t.string :name, index: true
 
       t.timestamps
     end

db/migrate/20170628014514_create_companies.rb

 class CreateCompanies < ActiveRecord::Migration[5.1]
   def change
     create_table :companies do |t|
-      t.string :name
+      t.string :name, index: true
       t.string :address
       t.text :description
       t.string :district

db/migrate/20170628014549_create_contacts.rb

@@ -2,7 +2,7 @@ class CreateContacts < ActiveRecord::Migration[5.1]
   def change
     create_table :contacts do |t|
       t.string :name
-      t.string :email
+      t.string :email, index: true
       t.string :phone
 
       t.timestamps

db/schema.rb

@@ -54,6 +54,7 @@ ActiveRecord::Schema.define(version: 20170628020034) do
     t.string "province"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["name"], name: "index_companies_on_name"
   end
 
   create_table "contacts", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
@@ -62,6 +63,7 @@ ActiveRecord::Schema.define(version: 20170628020034) do
     t.string "phone"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["email"], name: "index_contacts_on_email"
   end
 
   create_table "favorites", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
@@ -95,6 +97,7 @@ ActiveRecord::Schema.define(version: 20170628020034) do
     t.string "name"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["name"], name: "index_job_types_on_name"
   end
 
   create_table "jobs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|

lib/import/csv_reader.rb

+require 'thread'
+require 'logger'
+require 'csv'
+
+class Import::CSVReader
+  attr_reader :logger
+
+  def initialize(file)
+    @file = file
+    @logger = Logger.new("#{Rails.root}/log/csv_reader.log")
+  end
+
+  def import
+    @logger.info('Start read data')
+    puts '=======Start read data======='
+    csv_text = File.read(@file)
+    csv = CSV.parse(csv_text, headers: :true)
+    csv.each do |row|
+      begin
+        # Job type information
+        job_type = import_job_type(row)
+
+        # Contact information
+        contact = import_contact(row)
+
+        # Company information
+        company = import_company(row)
+
+        # Category information
+        category = import_category(row)
+
+        # City information
+        city = import_city(row)
+
+        # Job information
+        job = import_job(row, city, job_type, contact, company)
+
+        # Job Category Information
+        import_job_category(job, category)
+      rescue StandardError => e
+        logger.error(e.message)
+        logger.error(e.backtrace)
+        next
+      end
+    end
+    puts '=======End read data======='
+    @logger.info('End read data')
+  end
+
+  def import_job_type(row)
+    job_type_name = row['type'].strip unless row['type'].nil?
+    job_type = JobType.find_or_create_by(name: job_type_name)
+
+    job_type
+  end
+
+  def import_contact(row)
+    contact_name = row['contact name'].strip unless row['contact name'].nil?
+    contact_email = row['contact email'].strip unless row['contact email'].nil?
+    contact_phone = row['contact phone'].strip unless row['contact phone'].nil?
+
+    contact = Contact.find_or_create_by(email: contact_email)
+    contact.email = contact_email
+    contact.name = contact_name
+    contact.phone = contact_phone
+    contact.save
+
+    contact
+  end
+
+  def import_company(row)
+    company_address = row['company address'].strip unless row['company address'].nil?
+    company_district = row['company district'].strip unless row['company district'].nil?
+    company_name = row['company name'].strip unless row['company name'].nil?
+    company_province = row['company province'].strip unless row['company province'].nil?
+
+    company = Company.find_or_create_by(name: company_name)
+    company.address = company_address
+    company.district = company_district
+    company.name = company_name
+    company.province = company_province
+    company.save
+
+    company
+  end
+
+  def import_category(row)
+    category_name = row['category'].strip unless row['category'].nil?
+
+    category = Category.find_or_create_by(name: category_name)
+    category.name = category_name
+    category.save
+
+    category
+  end
+
+  def import_city(row)
+    city_name = row['work place'].strip unless row['work place'].nil?
+    city_name = city_name.tr('""', '').tr('[]', '') # Remove '["text"]' -> 'text'
+
+    city = City.find_or_create_by(name: city_name)
+    city.name = city_name
+    city.area = Area.find_by_name('Viet Nam')
+    city.save
+
+    city
+  end
+
+  def import_job(row, city, job_type, contact, company)
+    job_benefit = row['benefit'].strip unless row['benefit'].nil?
+    job_description = row['description'].strip unless row['description'].nil?
+    job_level = row['level'].strip unless row['level'].nil?
+    job_name = row['name'].strip unless row['name'].nil?
+    job_requirement = row['requirement'].strip unless row['requirement'].nil?
+    job_salary = row['salary'].strip unless row['salary'].nil?
+
+    job = Job.find_or_create_by(name: job_name, city: city, company: company)
+    job.benefit = job_benefit
+    job.description = job_description
+    job.level = job_level
+    job.name = job_name
+    job.requirement = job_requirement
+    job.salary = job_salary
+    job.city = city
+    job.job_type = job_type
+    job.contact = contact
+    job.company = company
+    job.save
+
+    job
+  end
+
+  def import_job_category(job, category)
+    job_category = JobCategory.find_or_create_by(job: job, category: category)
+    job_category.job = job
+    job_category.category = category
+    job_category.save
+  end
+end

lib/tasks/import_csv.rake

+require 'rubygems'
+
+namespace :import do
+  desc 'Import CSV'
+  task csv: :environment do
+
+    path_zip = "#{Rails.root}/lib/tasks/jobs.zip"
+    Utils::Download.new('192.168.1.156', 'training', 'training', path_zip, '*.zip').download_ftp
+
+    path_csv = "#{Rails.root}/lib/tasks/jobs.csv"
+    Zip::File.open(path_zip) do |zipfile|
+      zipfile.each do |entry|
+        entry.extract(path_csv)
+      end
+    end
+
+    Import::CSVReader.new(path_csv).import
+
+    File.delete(path_zip)
+    File.delete(path_csv)
+  end
+end

lib/utils/download.rb

+require 'net/ftp'
+require 'zip'
+
+class Utils::Download
+  attr_reader :url, :user, :password, :path, :file_type
+
+  def initialize(url, user, password, path, file_type)
+    @url = url
+    @user = user
+    @password = password
+    @path = path
+    @file_type = file_type
+  end
+
+  def download_ftp
+    ftp = Net::FTP.new
+    ftp.connect(@url)
+    ftp.login(@user, @password)
+    ftp.passive = true
+    files = ftp.nlst(@file_type)
+
+    files.each do |file_name|
+      ftp.getbinaryfile(file_name, @path)
+    end
+
+    ftp.close
+  end
+end