import data from ftp

Gemfile

@@ -23,6 +23,7 @@ gem 'kaminari'
 gem "settingslogic"
 gem "settingslogic"
 gem 'sidekiq'
 gem 'sidekiq'
 gem 'rsolr'
 gem 'rsolr'
+gem 'rubyzip'
 
 
 group :development, :test do
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console

Gemfile.lock

@@ -136,6 +136,7 @@ GEM
     rsolr (2.0.2)
     rsolr (2.0.2)
       builder (>= 2.1.2)
       builder (>= 2.1.2)
       faraday
       faraday
+    rubyzip (1.2.1)
     sass (3.5.3)
     sass (3.5.3)
       sass-listen (~> 4.0.0)
       sass-listen (~> 4.0.0)
     sass-listen (4.0.0)
     sass-listen (4.0.0)
@@ -193,6 +194,7 @@ DEPENDENCIES
   puma (~> 3.7)
   puma (~> 3.7)
   rails (~> 5.1.4)
   rails (~> 5.1.4)
   rsolr
   rsolr
+  rubyzip
   sass-rails (~> 5.0)
   sass-rails (~> 5.0)
   settingslogic
   settingslogic
   sidekiq
   sidekiq

app/data/import.rb

+require 'net/ftp'
+require 'csv'
+require 'zip'
+
+class Import
+  FTP_SERVER = '192.168.1.156'.freeze
+  USER = 'training'.freeze
+  PASS = 'training'.freeze
+  FILE = 'jobs.zip'.freeze
+  PATH_CSV = "#{Rails.root}/#{FILE}".freeze
+
+  def self.download_file
+    ftp = Net::FTP.new(FTP_SERVER, USER, PASS)
+    ftp.getbinaryfile(FILE)
+  end
+
+  def self.import_from_csv
+    params = []
+    zip_file = Zip::File.open(PATH_CSV)
+    entry = zip_file.glob('*.csv').first
+
+    csv_text = entry.get_input_stream.read
+    csv = CSV.parse(csv_text, headers: true)
+    csv.each do |row|
+      param = {}
+      # save job
+      param[:name] = encode(row['name'])
+      param[:salary] = encode(row['salary'])
+      param[:description] = encode(row['description'])
+      param[:level] = encode(row['level'])
+      param[:experience] = encode(row['requirement'])
+      param[:expiry_date] = nil
+      param[:updated_date] = nil
+      param[:updated_at] = nil
+
+      # save company
+      param[:company_name] = encode(row['company name'])
+      param[:company_location] = encode("#{row['company address']}, #{row['company district']}, #{row['company province']}".gsub(/,,/, ','))
+      param[:company_description] = encode(row['company description'])
+
+      # Save City
+      param[:city] = encode(row['company province'])
+
+      # Industry
+      param[:industry] = encode(row['category'])
+      params << param
+    end
+
+    params
+  end
+
+  def self.encode(str)
+    return str.force_encoding('UTF-8') unless str.nil?
+    nil
+  end
+end

db/migrate/20171101062514_change_data_type_for_experience.rb

+class ChangeDataTypeForExperience < ActiveRecord::Migration[5.1]
+  def change
+    change_column :jobs, :experience, :text
+  end
+end

db/migrate/20171101064311_change_data_type_for_location.rb

+class ChangeDataTypeForLocation < ActiveRecord::Migration[5.1]
+  def change
+    change_column :companies, :location, :text
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 #
 # It's strongly recommended that you check this file into your version control system.
 # It's strongly recommended that you check this file into your version control system.
 
 
-ActiveRecord::Schema.define(version: 20171031020435) do
+ActiveRecord::Schema.define(version: 20171101064311) do
 
 
   create_table "admins", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
   create_table "admins", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
     t.string "email", default: "", null: false
     t.string "email", default: "", null: false
@@ -67,7 +67,7 @@ ActiveRecord::Schema.define(version: 20171031020435) do
 
 
   create_table "companies", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
   create_table "companies", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
     t.string "name"
     t.string "name"
-    t.string "location"
+    t.text "location"
     t.text "description"
     t.text "description"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
@@ -110,7 +110,7 @@ ActiveRecord::Schema.define(version: 20171031020435) do
     t.string "salary"
     t.string "salary"
     t.text "description"
     t.text "description"
     t.string "level"
     t.string "level"
-    t.string "experience"
+    t.text "experience"
     t.bigint "company_id"
     t.bigint "company_id"
     t.datetime "expiry_date"
     t.datetime "expiry_date"
     t.datetime "updated_date"
     t.datetime "updated_date"

lib/tasks/data.rake

 require "./app/data/crawler.rb"
 require "./app/data/crawler.rb"
+# require './app/data/import.rb'
 
 
 namespace :data do
 namespace :data do
   task insert_job: :environment do |t|
   task insert_job: :environment do |t|
@@ -8,6 +9,17 @@ namespace :data do
     Job.create_new_jobs(@data)
     Job.create_new_jobs(@data)
   end
   end
 
 
+  desc 'insert job from csv file'
+  task import_job: :environment do |t|
+    @data = Import.import_from_csv
+    Job.create_new_jobs(@data)
+  end
+
+  desc 'download csv file from ftp'
+  task down_csv: :environment do |t|
+    Import.download_file
+  end
+
   desc "reset counter industry"
   desc "reset counter industry"
   task reset_counter_industry: :environment do |t|
   task reset_counter_industry: :environment do |t|
     Industry.find_each { |industry| Industry.reset_counters(industry.id, :jobs) }
     Industry.find_each { |industry| Industry.reset_counters(industry.id, :jobs) }