fix import data from csv

0f57f980 · Ngô Trung Hưng · 12437310 · 0f57f980 · 0f57f980 · 0f57f980
Commit 0f57f980 authored Jul 28, 2020 by Ngô Trung Hưng
Show whitespace changes
Inline Side-by-side

Showing with 40 additions and 38 deletions

lib/service/ftp.rb
+7 -4

lib/service/import_data.rb
+29 -32

lib/service/unzip.rb
+3 -1

lib/tasks/csv.rake
+1 -1

No files found.
--- a/lib/service/ftp.rb
+++ b/lib/service/ftp.rb
@@ -2,6 +2,7 @@
 require 'service/unzip'
 require 'net/ftp'
+require 'logger'
 require 'csv'
 # Access Ftp Server get file & extract file
@@ -9,15 +10,17 @@ class FtpSever
  CONTENT_SERVER_DOMAIN_NAME = '192.168.1.156'
  CONTENT_SERVER_USER_NAME = 'training'
  CONTENT_SERVER_USER_PASSWORD = 'training'
+  @logger ||= Logger.new("#{Rails.root}/log/csv.log")
  def self.donwload_csv
    Net::FTP.open(CONTENT_SERVER_DOMAIN_NAME, CONTENT_SERVER_USER_NAME, CONTENT_SERVER_USER_PASSWORD) do |ftp|
      begin
        ftp.getbinaryfile('jobs.zip')
-        extract_zip('./jobs.zip', 'lib/csv')
+        Unzip.extract_zip('./jobs.zip', 'lib/csv')
        File.delete('./jobs.zip') if File.exist?('./jobs.zip')
-        puts 'Extract file done'
+        @logger.info 'Donwload & extract success'
-      rescue StandardError => e
+      rescue FileNotFound => e
-        puts "#{e} File not found"
+        @logger.error "#{e.message}"
      end
    end
  end

--- a/lib/service/import_data.rb
+++ b/lib/service/import_data.rb
@@ -3,24 +3,24 @@
 require 'service/ftp'
 # Import data from CSV
-class ImportDataCSV
+class ImportData
  DOMESTIC = 1
  COMPANY_SECURITY = 1
  def import_data
    csv_data = FtpSever.data_csv
-    parse_csv_industries(csv_data)
+    import_industries_from(csv_data)
-    parse_csv_cities(csv_data)
+    import_cities_from(csv_data)
-    parse_csv_companies(csv_data)
+    import_companies_from(csv_data)
-    parse_csv_jobs(csv_data)
+    import_jobs_from(csv_data)
  end
  private
-  def parse_csv_industries(data)
+  def import_industries_from(csv)
    puts 'Import data industries . . .'
    industries = []
-    industries += data['category'].map(&:strip)
+    industries += csv['category'].map(&:strip)
    industries.each do |val|
      val.gsub!(',', '/') if val.include?(',')
      val.gsub!('/', ' / ')
@@ -29,9 +29,9 @@ class ImportDataCSV
    puts 'Done parse csv industries'
  end
-  def parse_csv_cities(data)
+  def import_cities_from(csv)
    puts 'Import data cities . . .'
-    cities = data['work place'].uniq.select(&:present?)
+    cities = csv['work place'].uniq.select(&:present?)
    cities = cities.map { |val| val.delete('[]\"') }
    cities.each do |val|
      next if val.blank?
@@ -40,12 +40,12 @@ class ImportDataCSV
    end
  end
-  def parse_csv_companies(data)
+  def import_companies_from(csv)
    puts 'Import data companies . . .'
-    data['company name'].each_with_index do |name, index|
+    csv['company name'].each_with_index do |name, index|
      Company.find_or_create_by(name: name.strip) do |company|
-        company.address           =  data['company address'][index]
+        company.address           =  csv['company address'][index]
-        company.short_description =  data['benefit'][index]
+        company.short_description =  csv['benefit'][index]
      end
      puts index
    rescue StandardError => e
@@ -53,22 +53,20 @@ class ImportDataCSV
    end
  end
-  def parse_csv_jobs(data)
+  def import_jobs_from(csv)
-    data['name'].each_with_index do |name, index|
+    csv['name'].each_with_index do |name, index|
-      desc = (data['requirement'][index]).to_s << (data['description'][index]).to_s
+      desc = "#{csv['requirement'][index]} #{(csv['description'][index])}"
-      id_company = Company.find_by name: data['company name'][index].to_s.strip
+      company = Company.find_by name: csv['company name'][index].to_s.strip
-      id_company = id_company.blank? ? COMPANY_SECURITY : id_company.id
+      id_company = company.blank? ? COMPANY_SECURITY : company.id
      begin
-        id_job = Job.create!(name: name,
+        job = Job.create!(name: name,
                          company_id: id_company,
-                             level: data['level'][index],
+                          level: csv['level'][index],
-                             experience: '',
+                          salary: csv['salary'][index],
-                             salary: data['salary'][index],
                          create_date: Time.now,
-                             expiration_date: '',
                          description: desc)
-        make_foreign_cities_table(data['work place'][index], id_job.id)
+        make_foreign_cities_table(csv['work place'][index], job.id)
-        make_foreign_industries_table(data['category'][index], id_job.id)
+        make_foreign_industries_table(csv['category'][index], job.id)
        puts index
      rescue StandardError => e
        puts e
@@ -78,16 +76,15 @@ class ImportDataCSV
  def make_foreign_cities_table(data, id_job)
    data = data.to_s.delete('[]\"')
-    id_city = City.find_by name: data.strip
+    city = City.find_or_create_by(name: data.strip) { |record| record.area = DOMESTIC }
-    id_city = id_city.blank? ? City.create!(name: data.strip, area: DOMESTIC).id : id_city.id
+    city_id = city.id
-    CityJob.create!(job_id: id_job, city_id: id_city)
+    CityJob.create!(job_id: id_job, city_id: city_id)
  end
  def make_foreign_industries_table(data, id_job)
    data = data.to_s.gsub(',', '/').gsub('/', ' / ')
-    data = data.strip
+    industry = Industry.find_or_create_by(name: data.strip)
-    id_industry = Industry.find_by name: data
+    industry_id = industry.id
-    id_industry = id_industry.blank? ? Industry.create!(name: data.strip).id : id_industry.id
+    IndustryJob.create!(industry_id: industry_id, job_id: id_job)
-    IndustryJob.create!(industry_id: id_industry, job_id: id_job)
  end
 end
--- a/lib/service/unzip.rb
+++ b/lib/service/unzip.rb
@@ -3,7 +3,8 @@
 require 'zip'
 # Description/Explanation of Person class
-def extract_zip(file, destination)
+module Unzip
+  def self.extract_zip(file, destination)
    FileUtils.mkdir_p(destination)
    Zip::File.open(file) do |zip_file|
      zip_file.each do |f|
@@ -11,4 +12,5 @@ def extract_zip(file, destination)
        zip_file.extract(f, fpath) unless File.exist?(fpath)
      end
    end
+  end
 end
--- a/lib/tasks/csv.rake
+++ b/lib/tasks/csv.rake
@@ -5,7 +5,7 @@ require 'service/import_data'
 # Description/Explanation of Person class
 namespace :csv do
  task import_csv: :environment do
-    csv = ImportDataCSV.new
+    csv = ImportData.new
    csv.import_data
  end
 end