Merge branch 'features/csv_import' into 'master'

Import data into db from ftp server

See merge request !9

Gemfile

@@ -22,6 +22,8 @@ gem 'carrierwave'
 gem 'kaminari'
 gem 'kaminari'
 gem "settingslogic"
 gem "settingslogic"
 gem 'sidekiq'
 gem 'sidekiq'
+gem 'rsolr'
+gem 'rubyzip'
 
 
 group :development, :test do
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console

Gemfile.lock

@@ -43,7 +43,7 @@ GEM
     bindex (0.5.0)
     bindex (0.5.0)
     builder (3.2.3)
     builder (3.2.3)
     byebug (9.1.0)
     byebug (9.1.0)
-    carrierwave (1.2.0)
+    carrierwave (1.2.1)
       activemodel (>= 4.0.0)
       activemodel (>= 4.0.0)
       activesupport (>= 4.0.0)
       activesupport (>= 4.0.0)
       mime-types (>= 1.16)
       mime-types (>= 1.16)
@@ -56,26 +56,29 @@ GEM
       railties (>= 4.1.0, < 5.2)
       railties (>= 4.1.0, < 5.2)
       responders
       responders
       warden (~> 1.2.3)
       warden (~> 1.2.3)
-    erubi (1.6.1)
+    erubi (1.7.0)
     execjs (2.7.0)
     execjs (2.7.0)
+    faraday (0.13.1)
+      multipart-post (>= 1.2, < 3)
     ffi (1.9.18)
     ffi (1.9.18)
     figaro (1.1.1)
     figaro (1.1.1)
       thor (~> 0.14)
       thor (~> 0.14)
-    globalid (0.4.0)
+    globalid (0.4.1)
       activesupport (>= 4.2.0)
       activesupport (>= 4.2.0)
-    i18n (0.8.6)
-    kaminari (1.0.1)
+    i18n (0.9.0)
+      concurrent-ruby (~> 1.0)
+    kaminari (1.1.1)
       activesupport (>= 4.1.0)
       activesupport (>= 4.1.0)
-      kaminari-actionview (= 1.0.1)
-      kaminari-activerecord (= 1.0.1)
-      kaminari-core (= 1.0.1)
-    kaminari-actionview (1.0.1)
+      kaminari-actionview (= 1.1.1)
+      kaminari-activerecord (= 1.1.1)
+      kaminari-core (= 1.1.1)
+    kaminari-actionview (1.1.1)
       actionview
       actionview
-      kaminari-core (= 1.0.1)
-    kaminari-activerecord (1.0.1)
+      kaminari-core (= 1.1.1)
+    kaminari-activerecord (1.1.1)
       activerecord
       activerecord
-      kaminari-core (= 1.0.1)
-    kaminari-core (1.0.1)
+      kaminari-core (= 1.1.1)
+    kaminari-core (1.1.1)
     loofah (2.1.1)
     loofah (2.1.1)
       crass (~> 1.0.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
       nokogiri (>= 1.5.9)
@@ -87,6 +90,7 @@ GEM
     mime-types-data (3.2016.0521)
     mime-types-data (3.2016.0521)
     mini_portile2 (2.3.0)
     mini_portile2 (2.3.0)
     minitest (5.10.3)
     minitest (5.10.3)
+    multipart-post (2.0.0)
     mysql2 (0.4.9)
     mysql2 (0.4.9)
     nio4r (2.1.0)
     nio4r (2.1.0)
     nokogiri (1.8.1)
     nokogiri (1.8.1)
@@ -121,7 +125,7 @@ GEM
       method_source
       method_source
       rake (>= 0.8.7)
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
       thor (>= 0.18.1, < 2.0)
-    rake (12.1.0)
+    rake (12.2.1)
     rb-fsevent (0.10.2)
     rb-fsevent (0.10.2)
     rb-inotify (0.9.10)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
       ffi (>= 0.5.0, < 2)
@@ -129,7 +133,11 @@ GEM
     responders (2.4.0)
     responders (2.4.0)
       actionpack (>= 4.2.0, < 5.3)
       actionpack (>= 4.2.0, < 5.3)
       railties (>= 4.2.0, < 5.3)
       railties (>= 4.2.0, < 5.3)
-    sass (3.5.1)
+    rsolr (2.0.2)
+      builder (>= 2.1.2)
+      faraday
+    rubyzip (1.2.1)
+    sass (3.5.3)
       sass-listen (~> 4.0.0)
       sass-listen (~> 4.0.0)
     sass-listen (4.0.0)
     sass-listen (4.0.0)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-fsevent (~> 0.9, >= 0.9.4)
@@ -158,7 +166,7 @@ GEM
     thor (0.20.0)
     thor (0.20.0)
     thread_safe (0.3.6)
     thread_safe (0.3.6)
     tilt (2.0.8)
     tilt (2.0.8)
-    tzinfo (1.2.3)
+    tzinfo (1.2.4)
       thread_safe (~> 0.1)
       thread_safe (~> 0.1)
     uglifier (3.2.0)
     uglifier (3.2.0)
       execjs (>= 0.3.0, < 3)
       execjs (>= 0.3.0, < 3)
@@ -185,6 +193,8 @@ DEPENDENCIES
   mysql2 (>= 0.3.18, < 0.5)
   mysql2 (>= 0.3.18, < 0.5)
   puma (~> 3.7)
   puma (~> 3.7)
   rails (~> 5.1.4)
   rails (~> 5.1.4)
+  rsolr
+  rubyzip
   sass-rails (~> 5.0)
   sass-rails (~> 5.0)
   settingslogic
   settingslogic
   sidekiq
   sidekiq

app/controllers/admins/applies_controller.rb

+class Admins::AppliesController < ApplicationController
+  def index
+  end
+end

app/controllers/application_controller.rb

@@ -5,4 +5,13 @@ class ApplicationController < ActionController::Base
   def clear_session_candidate
   def clear_session_candidate
     session[:candidate] = {}
     session[:candidate] = {}
   end
   end
+
+  def after_sign_in_path_for(resource)
+    stored_location_for(resource) ||
+      if resource.is_a?(Admin)
+        admins_applies_path
+      else
+        super
+      end
+  end
 end
 end

app/data/import.rb

+require 'net/ftp'
+require 'csv'
+require 'zip'
+
+class Import
+  FTP_SERVER = '192.168.1.156'.freeze
+  FILE = 'jobs.zip'.freeze
+  PATH_CSV = "#{Rails.root}/#{FILE}".freeze
+
+  def self.download_file
+    ftp = Net::FTP.new(FTP_SERVER, ENV["USER_FTP"], ENV["PASS_FTP"])
+    ftp.getbinaryfile(FILE)
+  end
+
+  def self.import_from_csv
+    params = []
+    zip_file = Zip::File.open(PATH_CSV)
+    entry = zip_file.glob('*.csv').first
+
+    csv_text = entry.get_input_stream.read
+    csv = CSV.parse(csv_text, headers: true)
+    csv.each do |row|
+      param = {}
+      # save job
+      param[:name] = encode(row['name'])
+      param[:salary] = encode(row['salary'])
+      param[:description] = encode(row['description'])
+      param[:level] = encode(row['level'])
+      param[:experience] = encode(row['requirement'])
+      param[:expiry_date] = nil
+      param[:updated_date] = nil
+      param[:updated_at] = nil
+
+      # save company
+      param[:company_name] = encode(row['company name'])
+      param[:company_location] = encode("#{row['company address']}, #{row['company district']}, #{row['company province']}".gsub(/,,/, ','))
+      param[:company_description] = encode(row['company description'])
+
+      # Save City
+      param[:city] = encode(row['company province'])
+
+      # Industry
+      param[:industry] = encode(row['category'])
+      params << param
+    end
+
+    params
+  end
+
+  def self.encode(str)
+    return str.force_encoding('UTF-8') unless str.nil?
+    nil
+  end
+end

app/models/admin.rb

+class Admin < ApplicationRecord
+  # Include default devise modules. Others available are:
+  # :confirmable, :lockable, :timeoutable and :omniauthable
+  devise :database_authenticatable, :registerable,
+         :recoverable, :rememberable, :trackable, :validatable
+end

app/views/admins/applies/index.html.erb

+<h1>Admins::Applies#index</h1>
+<p>Find me in app/views/admins/applies/index.html.erb</p>

app/views/admins/sessions/new.html.erb

+<div class="row">
+  <div class="container">
+    <h2>Admin login</h2>
+    <%= form_for(resource, as: resource_name, url: session_path(resource_name)) do |f| %>
+    <div class="field form-group">
+      <%= f.label :email %><br />
+      <%= f.email_field :email, autofocus: true, class: "form-control" %>
+    </div>
+
+    <div class="field form-group">
+      <%= f.label :password %><br />
+      <%= f.password_field :password, autocomplete: "off", class: "form-control" %>
+    </div>
+
+    <% if devise_mapping.rememberable? -%>
+    <div class="field form-group">
+      <%= f.check_box :remember_me %>
+      <%= f.label :remember_me %>
+    </div>
+    <% end -%>
+
+    <div class="actions form-group">
+      <%= f.submit "Log in", class: "btn btn-default" %>
+    </div>
+    <% end %>
+  </div>
+</div>

config/routes.rb

 Rails.application.routes.draw do
 Rails.application.routes.draw do
   require 'sidekiq/web'
   require 'sidekiq/web'
+  devise_for :admins
+  namespace :admins do
+    resources :applies, only: [:index]
+    root 'applies#index'
+  end
   root 'jobs#index'
   root 'jobs#index'
   devise_for :users, :controllers => {:registrations => "registrations"}
   devise_for :users, :controllers => {:registrations => "registrations"}
   as :user do
   as :user do
     get 'register/:step', to: 'registrations#new', as: :register
     get 'register/:step', to: 'registrations#new', as: :register
     patch 'complete_registering' => 'registrations#complete_registering_user'
     patch 'complete_registering' => 'registrations#complete_registering_user'
   end
   end
-
   resources :cities, only: [:index, :show]
   resources :cities, only: [:index, :show]
   resources :industries, only: [:index, :show]
   resources :industries, only: [:index, :show]
   resources :favorites do
   resources :favorites do

db/migrate/20171031020435_devise_create_admins.rb

+class DeviseCreateAdmins < ActiveRecord::Migration[5.1]
+  def change
+    create_table :admins do |t|
+      ## Database authenticatable
+      t.string :email,              null: false, default: ""
+      t.string :encrypted_password, null: false, default: ""
+
+      ## Recoverable
+      t.string   :reset_password_token
+      t.datetime :reset_password_sent_at
+
+      ## Rememberable
+      t.datetime :remember_created_at
+
+      ## Trackable
+      t.integer  :sign_in_count, default: 0, null: false
+      t.datetime :current_sign_in_at
+      t.datetime :last_sign_in_at
+      t.string   :current_sign_in_ip
+      t.string   :last_sign_in_ip
+
+      ## Confirmable
+      # t.string   :confirmation_token
+      # t.datetime :confirmed_at
+      # t.datetime :confirmation_sent_at
+      # t.string   :unconfirmed_email # Only if using reconfirmable
+
+      ## Lockable
+      # t.integer  :failed_attempts, default: 0, null: false # Only if lock strategy is :failed_attempts
+      # t.string   :unlock_token # Only if unlock strategy is :email or :both
+      # t.datetime :locked_at
+
+
+      t.timestamps null: false
+    end
+
+    add_index :admins, :email,                unique: true
+    add_index :admins, :reset_password_token, unique: true
+    # add_index :admins, :confirmation_token,   unique: true
+    # add_index :admins, :unlock_token,         unique: true
+  end
+end

db/migrate/20171101062514_change_data_type_for_experience.rb

+class ChangeDataTypeForExperience < ActiveRecord::Migration[5.1]
+  def change
+    change_column :jobs, :experience, :text
+  end
+end

db/migrate/20171101064311_change_data_type_for_location.rb

+class ChangeDataTypeForLocation < ActiveRecord::Migration[5.1]
+  def change
+    change_column :companies, :location, :text
+  end
+end

db/schema.rb

@@ -10,7 +10,24 @@
 #
 #
 # It's strongly recommended that you check this file into your version control system.
 # It's strongly recommended that you check this file into your version control system.
 
 
-ActiveRecord::Schema.define(version: 20171018012604) do
+ActiveRecord::Schema.define(version: 20171101064311) do
+
+  create_table "admins", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
+    t.string "email", default: "", null: false
+    t.string "encrypted_password", default: "", null: false
+    t.string "reset_password_token"
+    t.datetime "reset_password_sent_at"
+    t.datetime "remember_created_at"
+    t.integer "sign_in_count", default: 0, null: false
+    t.datetime "current_sign_in_at"
+    t.datetime "last_sign_in_at"
+    t.string "current_sign_in_ip"
+    t.string "last_sign_in_ip"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["email"], name: "index_admins_on_email", unique: true
+    t.index ["reset_password_token"], name: "index_admins_on_reset_password_token", unique: true
+  end
 
 
   create_table "apply_jobs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
   create_table "apply_jobs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
     t.bigint "job_id"
     t.bigint "job_id"
@@ -50,7 +67,7 @@ ActiveRecord::Schema.define(version: 20171018012604) do
 
 
   create_table "companies", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
   create_table "companies", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
     t.string "name"
     t.string "name"
-    t.string "location"
+    t.text "location"
     t.text "description"
     t.text "description"
     t.datetime "created_at", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "updated_at", null: false
@@ -93,7 +110,7 @@ ActiveRecord::Schema.define(version: 20171018012604) do
     t.string "salary"
     t.string "salary"
     t.text "description"
     t.text "description"
     t.string "level"
     t.string "level"
-    t.string "experience"
+    t.text "experience"
     t.bigint "company_id"
     t.bigint "company_id"
     t.datetime "expiry_date"
     t.datetime "expiry_date"
     t.datetime "updated_date"
     t.datetime "updated_date"
@@ -126,4 +143,13 @@ ActiveRecord::Schema.define(version: 20171018012604) do
     t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
     t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
   end
   end
 
 
+  create_table "view_jobs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
+    t.bigint "user_id"
+    t.bigint "job_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["job_id"], name: "index_view_jobs_on_job_id"
+    t.index ["user_id"], name: "index_view_jobs_on_user_id"
+  end
+
 end
 end

lib/tasks/data.rake

 require "./app/data/crawler.rb"
 require "./app/data/crawler.rb"
+# require './app/data/import.rb'
 
 
 namespace :data do
 namespace :data do
   task insert_job: :environment do |t|
   task insert_job: :environment do |t|
@@ -8,6 +9,17 @@ namespace :data do
     Job.create_new_jobs(@data)
     Job.create_new_jobs(@data)
   end
   end
 
 
+  desc 'insert job from csv file'
+  task import_job: :environment do |t|
+    @data = Import.import_from_csv
+    Job.create_new_jobs(@data)
+  end
+
+  desc 'download csv file from ftp'
+  task down_csv: :environment do |t|
+    Import.download_file
+  end
+
   desc "reset counter industry"
   desc "reset counter industry"
   task reset_counter_industry: :environment do |t|
   task reset_counter_industry: :environment do |t|
     Industry.find_each { |industry| Industry.reset_counters(industry.id, :jobs) }
     Industry.find_each { |industry| Industry.reset_counters(industry.id, :jobs) }