Merge branch 'csv' into 'master'

create import csv function

See merge request !3

.gitignore

@@ -25,3 +25,4 @@
 
 
 # Ignore master key for decrypting credentials and more.
 # Ignore master key for decrypting credentials and more.
 /config/master.key
 /config/master.key
+/lib/csv

app/models/city.rb

@@ -5,6 +5,8 @@ class City < ApplicationRecord
   RANGE = 69
   RANGE = 69
   has_many :city_jobs
   has_many :city_jobs
   has_many :jobs, through: :city_jobs  
   has_many :jobs, through: :city_jobs  
+  enum area: { international: 0, domestic:1}
+  has_many :jobs, through: :city_jobs
   scope :domestic, -> { where(area: 1) }
   scope :domestic, -> { where(area: 1) }
   scope :international, -> { where(area: 0) }
   scope :international, -> { where(area: 0) }
   scope :all_cities, -> { select :id, :name }
   scope :all_cities, -> { select :id, :name }

config/application.rb

@@ -10,6 +10,7 @@ module Venjob
   class Application < Rails::Application
   class Application < Rails::Application
     # Initialize configuration defaults for originally generated Rails version.
     # Initialize configuration defaults for originally generated Rails version.
     config.load_defaults 5.2
     config.load_defaults 5.2
+    config.autoload_paths << Rails.root.join('lib/service')
     config.autoload_paths << Rails.root.join('lib/src')
     config.autoload_paths << Rails.root.join('lib/src')
     # Settings in config/environments/* take precedence over those specified here.
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration can go into files in config/initializers
     # Application configuration can go into files in config/initializers

config/database.yml

@@ -11,7 +11,7 @@
 #
 #
 default: &default
 default: &default
   adapter: mysql2
   adapter: mysql2
-  encoding: utf8
+  encoding: utf8mb4
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   username: root
   username: root
   password: '1'
   password: '1'

db/migrate/20200728020937_create_companies.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateCompanies < ActiveRecord::Migration[5.2]
 class CreateCompanies < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :companies, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :companies do |t|
       t.string :name
       t.string :name
       t.string :address
       t.string :address
       t.text :short_description
       t.text :short_description

db/migrate/20200728021021_create_cities.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateCities < ActiveRecord::Migration[5.2]
 class CreateCities < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :cities, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :cities do |t|
       t.string :name
       t.string :name
       t.boolean :area
       t.boolean :area
       t.timestamps
       t.timestamps

db/migrate/20200728021030_create_industries.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateIndustries < ActiveRecord::Migration[5.2]
 class CreateIndustries < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :industries, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :industries do |t|
       t.string :name
       t.string :name
       t.timestamps
       t.timestamps
     end
     end

db/migrate/20200728021038_create_jobs.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateJobs < ActiveRecord::Migration[5.2]
 class CreateJobs < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :jobs, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :jobs do |t|
       t.string :name
       t.string :name
       t.integer :company_id
       t.integer :company_id
       t.string :level
       t.string :level

db/migrate/20200728021048_create_histories.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateHistories < ActiveRecord::Migration[5.2]
 class CreateHistories < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :histories, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :histories do |t|
       t.integer :user_id
       t.integer :user_id
       t.integer :job_id
       t.integer :job_id
       t.timestamps
       t.timestamps

db/migrate/20200728021110_create_favorites.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateFavorites < ActiveRecord::Migration[5.2]
 class CreateFavorites < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :favorites, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :favorites do |t|
       t.integer :user_id
       t.integer :user_id
       t.integer :job_id
       t.integer :job_id
       t.timestamps
       t.timestamps

db/migrate/20200728021127_create_users.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateUsers < ActiveRecord::Migration[5.2]
 class CreateUsers < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :users, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :users do |t|
       t.string :email
       t.string :email
       t.string :name
       t.string :name
       t.string :password_digest
       t.string :password_digest

db/migrate/20200728021254_create_city_jobs.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateCityJobs < ActiveRecord::Migration[5.2]
 class CreateCityJobs < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :city_jobs, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :city_jobs do |t|
       t.references :job
       t.references :job
       t.references :city
       t.references :city
       t.timestamps
       t.timestamps

db/migrate/20200728021350_create_industry_jobs.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateIndustryJobs < ActiveRecord::Migration[5.2]
 class CreateIndustryJobs < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :industry_jobs, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :industry_jobs do |t|
       t.references :industry
       t.references :industry
       t.references :job
       t.references :job
       t.timestamps
       t.timestamps

db/migrate/20200728021412_create_applied_jobs.rb

@@ -3,7 +3,7 @@
 # Description/Explanation of Person class
 # Description/Explanation of Person class
 class CreateAppliedJobs < ActiveRecord::Migration[5.2]
 class CreateAppliedJobs < ActiveRecord::Migration[5.2]
   def change
   def change
-    create_table :applied_jobs, options: 'COLLATE=utf8_general_ci' do |t|
+    create_table :applied_jobs do |t|
       t.references :user
       t.references :user
       t.references :job
       t.references :job
       t.string :name
       t.string :name

db/schema.rb

@@ -12,7 +12,7 @@
 
 
 ActiveRecord::Schema.define(version: 2020_07_29_064551) do
 ActiveRecord::Schema.define(version: 2020_07_29_064551) do
 
 
-  create_table "applied_jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "applied_jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.bigint "user_id"
     t.bigint "user_id"
     t.bigint "job_id"
     t.bigint "job_id"
     t.string "name"
     t.string "name"
@@ -24,14 +24,14 @@ ActiveRecord::Schema.define(version: 2020_07_29_064551) do
     t.index ["user_id"], name: "index_applied_jobs_on_user_id"
     t.index ["user_id"], name: "index_applied_jobs_on_user_id"
   end
   end
 
 
-  create_table "cities", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "cities", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.string "name"
     t.string "name"
     t.integer "area"
     t.integer "area"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
   end
   end
 
 
-  create_table "city_jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "city_jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.bigint "job_id"
     t.bigint "job_id"
     t.bigint "city_id"
     t.bigint "city_id"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
@@ -40,7 +40,7 @@ ActiveRecord::Schema.define(version: 2020_07_29_064551) do
     t.index ["job_id"], name: "index_city_jobs_on_job_id"
     t.index ["job_id"], name: "index_city_jobs_on_job_id"
   end
   end
 
 
-  create_table "companies", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "companies", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.string "name"
     t.string "name"
     t.string "address"
     t.string "address"
     t.text "short_description"
     t.text "short_description"
@@ -48,27 +48,27 @@ ActiveRecord::Schema.define(version: 2020_07_29_064551) do
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
   end
   end
 
 
-  create_table "favorites", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "favorites", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.integer "user_id"
     t.integer "user_id"
     t.integer "job_id"
     t.integer "job_id"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
   end
   end
 
 
-  create_table "histories", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "histories", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.integer "user_id"
     t.integer "user_id"
     t.integer "job_id"
     t.integer "job_id"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
   end
   end
 
 
-  create_table "industries", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "industries", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.string "name"
     t.string "name"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
   end
   end
 
 
-  create_table "industry_jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "industry_jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.bigint "industry_id"
     t.bigint "industry_id"
     t.bigint "job_id"
     t.bigint "job_id"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
@@ -77,7 +77,7 @@ ActiveRecord::Schema.define(version: 2020_07_29_064551) do
     t.index ["job_id"], name: "index_industry_jobs_on_job_id"
     t.index ["job_id"], name: "index_industry_jobs_on_job_id"
   end
   end
 
 
-  create_table "jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "jobs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.string "name"
     t.string "name"
     t.integer "company_id"
     t.integer "company_id"
     t.string "level"
     t.string "level"
@@ -90,7 +90,7 @@ ActiveRecord::Schema.define(version: 2020_07_29_064551) do
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
   end
   end
 
 
-  create_table "users", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "users", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t|
     t.string "email"
     t.string "email"
     t.string "name"
     t.string "name"
     t.string "password_digest"
     t.string "password_digest"

lib/service/ftp_sever.rb

+# frozen_string_literal: true
+
+require 'net/ftp'
+require 'logger'
+require 'csv'
+
+# Access Ftp Server get file & extract file
+class FtpSever
+  CONTENT_SERVER_DOMAIN_NAME = '192.168.1.156'
+  CONTENT_SERVER_USER_NAME = 'training'
+  CONTENT_SERVER_USER_PASSWORD = 'training'
+  NAME_CSV = 'jobs.zip'
+
+  def data_csv
+    download_csv
+    CSV.parse(File.read(Rails.root.join('lib', 'csv', 'jobs.csv')), headers: true)
+  end
+
+  def logger
+    @logger ||= Logger.new(Rails.root.join('log', 'csv.log'))
+  end
+
+  private
+
+  def jobs_csv_path
+    Rails.root.join(NAME_CSV)
+  end
+
+  def download_csv
+    Net::FTP.open(CONTENT_SERVER_DOMAIN_NAME, CONTENT_SERVER_USER_NAME, CONTENT_SERVER_USER_PASSWORD) do |ftp|
+      ftp.getbinaryfile(NAME_CSV)
+      Unzip.extract_zip(jobs_csv_path, Rails.root.join('lib', 'csv'))
+      File.delete(jobs_csv_path) if File.exist?(jobs_csv_path)
+      logger.info 'Download & extract success'
+    rescue StandardError => e
+      logger.error "Donwload csv have error: #{e}"
+    end
+  end
+end

lib/service/import_data.rb

+# frozen_string_literal: true
+
+# Import data from CSV
+class ImportData
+  attr_accessor :csv
+
+  def initialize(csv)
+    @csv = csv
+  end
+
+  def import_data
+    import_companies
+    import_jobs
+  end
+
+  def logger
+    @logger ||= Logger.new(Rails.root.join('log', 'import_data_csv.log'))
+  end
+
+  private
+
+  def import_companies
+    csv['company name'].each_with_index do |name, index|
+      Company.find_or_create_by(name: name.strip) do |company|
+        company.address           =  csv['company address'][index]
+        company.short_description =  csv['benefit'][index]
+      end
+    rescue StandardError => e
+      logger.error "Import_companies: #{e}"
+    end
+  end
+
+  def import_jobs
+    csv.each do |val|
+      desc = "#{val['requirement']} #{(val['description'])}"
+      # Get ID Company to create job
+      company = Company.find_by name: val['company name'].strip
+      company_id = company.try(:id) || Company::COMPANY_SECURITY
+      # Get data city do create relationship city_jobs
+      city_names = val['work place'].delete('[]\"')
+      city = City.find_or_create_by(name: city_names.strip) { |record| record.area = City.areas['domestic'] }
+      # Get data Industry do create relationship industry_jobs
+      industry_name = val['category'].gsub(',', '/').gsub('/', ' / ')
+      industry = Industry.find_or_create_by(name: industry_name.strip)
+
+      Job.find_or_create_by(name: val['name'], company_id: company_id) do |job|
+        job.level = val['level']
+        job.salary = val['salary']
+        job.description = desc
+        job.city_ids.blank? ? job.cities << city : job.cities == city
+        job.industry_ids.blank? ? job.industries << industry : job.industries == industry
+      end
+    rescue StandardError => e
+      logger.error "Import_jobs: #{e}"
+    end
+  end
+end

lib/service/unzip.rb

+# frozen_string_literal: true
+
+require 'zip'
+
+# Description/Explanation of Person class
+module Unzip
+  def self.extract_zip(file, destination)
+    FileUtils.mkdir_p(destination)
+    Zip::File.open(file) do |zip_file|
+      zip_file.each do |f|
+        fpath = File.join(destination, f.name)
+        zip_file.extract(f, fpath) unless File.exist?(fpath)
+      end
+    end
+  end
+end

lib/tasks/csv.rake

+# frozen_string_literal: true
+
+# Import data from csv
+namespace :csv do
+  task import_csv: :environment do
+    data_csv = FtpSever.new.data_csv
+    ImportData.new(data_csv).import_data
+  end
+end