Feature import csv

Gemfile

@@ -51,6 +51,7 @@ group :development do
   gem 'spring-watcher-listen', '~> 2.0.0'
   gem 'better_errors'
   gem 'binding_of_caller'
+  gem 'rubyzip'
 end
 
 # Windows does not include zoneinfo files, so bundle the tzinfo-data gem

Gemfile.lock

@@ -194,6 +194,7 @@ DEPENDENCIES
   mysql2 (>= 0.3.18, < 0.5)
   puma (~> 3.7)
   rails (~> 5.1.1)
+  rubyzip
   sass-rails (~> 5.0)
   selenium-webdriver
   spring

db/migrate/20170628014416_create_job_types.rb

 class CreateJobTypes < ActiveRecord::Migration[5.1]
   def change
     create_table :job_types do |t|
-      t.string :name
+      t.string :name, index: true
 
       t.timestamps
     end

db/migrate/20170628014514_create_companies.rb

 class CreateCompanies < ActiveRecord::Migration[5.1]
   def change
     create_table :companies do |t|
-      t.string :name
+      t.string :name, index: true
       t.string :address
       t.text :description
       t.string :district

db/migrate/20170628014549_create_contacts.rb

@@ -2,7 +2,7 @@ class CreateContacts < ActiveRecord::Migration[5.1]
   def change
     create_table :contacts do |t|
       t.string :name
-      t.string :email
+      t.string :email, index: true
       t.string :phone
 
       t.timestamps

db/schema.rb

@@ -54,6 +54,7 @@ ActiveRecord::Schema.define(version: 20170628020034) do
     t.string "province"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["name"], name: "index_companies_on_name"
   end
 
   create_table "contacts", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
@@ -62,6 +63,7 @@ ActiveRecord::Schema.define(version: 20170628020034) do
     t.string "phone"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["email"], name: "index_contacts_on_email"
   end
 
   create_table "favorites", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
@@ -95,6 +97,7 @@ ActiveRecord::Schema.define(version: 20170628020034) do
     t.string "name"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["name"], name: "index_job_types_on_name"
   end
 
   create_table "jobs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|

lib/tasks/csv_reader.rb

+require 'thread'
+require 'logger'
+require 'csv'
+
+class CSVReader
+  attr_reader :thread_count, :logger
+
+  def initialize(file, thread_count)
+    @file = file
+    @thread_count = thread_count
+    @mutex = Mutex.new
+    @logger = Logger.new("#{Rails.root}/log/csv_reader.log")
+  end
+
+  def import
+    @logger.info('Start read data')
+    workers = (0...thread_count).map do
+      Thread.new do
+        begin
+          begin
+            csv_text = File.read(@file)
+            csv = CSV.parse(csv_text, headers: :true)
+            csv.each do |row|
+
+              job_type = JobType.new
+              job_type.name = row['type']
+              job_type.save if JobType.find_by_name(row['type']).blank?
+
+              contact = Contact.new
+              contact.email = row['contact email']
+              contact.name = row['contact name']
+              contact.phone = row['contact phone']
+              contact.save if Contact.find_by_email(row['email']).blank?
+
+              company = Company.new
+              company.address = row['company address']
+              company.district = row['company district']
+              company.name = row['company name']
+              company.province = row['company province']
+              company.save if Company.find_by_name(row['name']).blank?
+
+              category = Category.new
+              category.name = row['category']
+              category.save if Category.find_by_name(row['name']).blank?
+
+              job = Job.new
+              job.benefit = row['benefit']
+              job.description = row['description']
+              job.level = row['level']
+              job.name = row['name']
+              job.requirement = row['requirement']
+              job.salary = row['salary']
+              job.city = City.find_by_name(row['work place'])
+              job.job_type = job_type
+              job.contact = contact
+              job.company = company
+              job.save
+
+              job_category = JobCategory.new
+              job_category.job = job
+              job_category.category = category
+              job_category.save
+            end
+          rescue StandardError => e
+            logger.error(e.message)
+            logger.error(e.backtrace)
+          end
+          puts '=======Thread End======='
+        rescue ThreadError
+        end
+      end
+    end
+
+    workers.map(&:join)
+    @logger.info('End read data')
+  end
+end

lib/tasks/import_csv.rake

+require 'net/ftp'
+require 'rubygems'
+require 'zip'
+
+namespace :import do
+  desc 'Import CSV'
+  task csv: :environment do
+    require "#{Rails.root}/lib/tasks/csv_reader"
+    thread_count = ENV['THREAD_COUNT'] || 1
+
+    ftp = Net::FTP.new
+    ftp.connect('192.168.1.156')
+    ftp.login('training', 'training')
+    ftp.passive = true
+    files = ftp.nlst('*.zip')
+
+    path_zip = "#{Rails.root}/lib/tasks/jobs.zip"
+    files.each do |file_name|
+      ftp.getbinaryfile(file_name, path_zip)
+    end
+
+    ftp.close
+
+    path_csv = "#{Rails.root}/lib/tasks/jobs.csv"
+    Zip::File.open(path_zip) do |zipfile|
+      zipfile.each do |entry|
+        entry.extract(path_csv)
+      end
+    end
+
+    CSVReader.new(path_csv, thread_count).import
+
+    File.delete(path_zip)
+    File.delete(path_csv)
+  end
+end