Skip to content

V
veNJOB
Commit 4cbf054c by Thanh Hung Pham
Options

Merge remote-tracking branch 'origin/master'

parent ce3dbcdc
Showing with 404 additions and 33 deletions
Gemfile.lock
...@@ -210,6 +210,7 @@ DEPENDENCIES ...@@ -210,6 +210,7 @@ DEPENDENCIES
mysql2 (>= 0.3.18, < 0.5) mysql2 (>= 0.3.18, < 0.5)
puma (~> 3.7) puma (~> 3.7)
rails (~> 5.1.1) rails (~> 5.1.1)
rubyzip
sass-rails (~> 5.0) sass-rails (~> 5.0)
selenium-webdriver selenium-webdriver
spring spring
......
app/assets/javascripts/sessions.coffee 0 → 100644
# Place all the behaviors and hooks related to the matching controller here.
# All this logic will automatically be available in application.js.
# You can use CoffeeScript in this file: http://coffeescript.org/
app/assets/stylesheets/custom.scss
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
$eee: #eee; $eee: #eee;
$color888: #888; $color888: #888;
$bbb: #bbb;
body { body {
padding-top: 60px; padding-top: 60px;
...@@ -15,6 +16,13 @@ footer { ...@@ -15,6 +16,13 @@ footer {
} }
} }
input {
border: 1px solid $bbb;
height: auto;
margin-bottom: 15px;
width: 100%;
}
.navbar-inverse { .navbar-inverse {
background-color: $eee; background-color: $eee;
border-color: $color888; border-color: $color888;
......
app/assets/stylesheets/sessions.scss 0 → 100644
// Place all the styles related to the Sessions controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
app/controllers/sessions_controller.rb 0 → 100644
class SessionsController < ApplicationController
def new
end
def create
render 'new'
end
end
app/helpers/sessions_helper.rb 0 → 100644
module SessionsHelper
end
app/views/sessions/new.html.erb 0 → 100644
<%- provide(:title, 'Login') -%>
<div class="well">
<h2>Log in</h2>
<%= form_for(:session, url: login_path) do |f| %>
<div class="row">
<div class="col-md-2 col-md-offset-3">
<%= f.label :email %>
</div>
<div class="col-md-6">
<%= f.email_field :email, size: 50, maxlength: 255, class: 'form_control' %>
</div>
</div>
<div class="row">
<div class="col-md-2 col-md-offset-3">
<%= f.label :password %>
</div>
<div class="col-md-6">
<%= f.password_field :password, size: 50, maxlength: 255, class: 'form_control' %>
</div>
</div>
<div class="row">
<div class="col-md-6 col-md-offset-5">
<%= link_to 'Forgot password?', '#' %>
</div>
</div>
<div class="row">
<div class="col-md-6 col-md-offset-5">
<%= f.submit 'Log in', class: 'btn btn-primary' %>
<%= link_to 'Register', register_path, class: 'btn btn-primary' %>
</div>
</div>
<%- end -%>
</div>
config/application.rb
...@@ -14,5 +14,7 @@ module VeNJOB ...@@ -14,5 +14,7 @@ module VeNJOB
# Settings in config/environments/* take precedence over those specified here. # Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers # Application configuration should go into files in config/initializers
# -- all .rb files in that directory are automatically loaded. # -- all .rb files in that directory are automatically loaded.
config.autoload_paths += %W[#{config.root}/lib]
config.eager_load_paths += %W[#{config.root}/lib]
end end
end end
config/schedule.rb
...@@ -18,6 +18,11 @@ ...@@ -18,6 +18,11 @@
# end # end
# Learn more: http://github.com/javan/whenever # Learn more: http://github.com/javan/whenever
env :PATH, ENV['PATH']
set :environment, 'development'
set :output, { error: 'log/cron_error_log.log', standard: 'log/cron_log.log' }
every 1.day, at: '12:00 pm' do every 1.day, at: '12:00 pm' do
runner 'MyModel.task_to_run' rake 'crawler:load'
rake 'import:csv'
end end
db/migrate/20170628014416_create_job_types.rb
class CreateJobTypes < ActiveRecord::Migration[5.1] class CreateJobTypes < ActiveRecord::Migration[5.1]
def change def change
create_table :job_types do |t| create_table :job_types do |t|
t.string :name t.string :name, index: true
t.timestamps t.timestamps
end end
......
db/migrate/20170628014514_create_companies.rb
class CreateCompanies < ActiveRecord::Migration[5.1] class CreateCompanies < ActiveRecord::Migration[5.1]
def change def change
create_table :companies do |t| create_table :companies do |t|
t.string :name t.string :name, index: true
t.string :address t.string :address
t.text :description t.text :description
t.string :district t.string :district
......
db/migrate/20170628014549_create_contacts.rb
...@@ -2,7 +2,7 @@ class CreateContacts < ActiveRecord::Migration[5.1] ...@@ -2,7 +2,7 @@ class CreateContacts < ActiveRecord::Migration[5.1]
def change def change
create_table :contacts do |t| create_table :contacts do |t|
t.string :name t.string :name
t.string :email t.string :email, index: true
t.string :phone t.string :phone
t.timestamps t.timestamps
......
db/schema.rb
...@@ -54,6 +54,7 @@ ActiveRecord::Schema.define(version: 20170705034837) do ...@@ -54,6 +54,7 @@ ActiveRecord::Schema.define(version: 20170705034837) do
t.string "province" t.string "province"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.index ["name"], name: "index_companies_on_name"
end end
create_table "contacts", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t| create_table "contacts", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
...@@ -62,6 +63,7 @@ ActiveRecord::Schema.define(version: 20170705034837) do ...@@ -62,6 +63,7 @@ ActiveRecord::Schema.define(version: 20170705034837) do
t.string "phone" t.string "phone"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.index ["email"], name: "index_contacts_on_email"
end end
create_table "favorites", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t| create_table "favorites", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
...@@ -95,6 +97,7 @@ ActiveRecord::Schema.define(version: 20170705034837) do ...@@ -95,6 +97,7 @@ ActiveRecord::Schema.define(version: 20170705034837) do
t.string "name" t.string "name"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.index ["name"], name: "index_job_types_on_name"
end end
create_table "jobs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t| create_table "jobs", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
......
db/seeds.rb
...@@ -5,3 +5,5 @@ ...@@ -5,3 +5,5 @@
# #
# movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }]) # movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }])
# Character.create(name: 'Luke', movie: movies.first) # Character.create(name: 'Luke', movie: movies.first)
Area.new(name: 'Viet Nam').save
Area.new(name: 'International').save
lib/tasks/careerbuilder.rb → lib/crawler/careerbuilder.rb
...@@ -3,7 +3,7 @@ require 'open-uri' ...@@ -3,7 +3,7 @@ require 'open-uri'
require 'nokogiri' require 'nokogiri'
require 'logger' require 'logger'
class Careerbuilder class Crawler::Careerbuilder
attr_reader :domain, :thread_count, :logger attr_reader :domain, :thread_count, :logger
def initialize(domain, thread_count = 1) def initialize(domain, thread_count = 1)
...@@ -18,8 +18,7 @@ class Careerbuilder ...@@ -18,8 +18,7 @@ class Careerbuilder
def crawl def crawl
@logger.info('Start crawl') @logger.info('Start crawl')
doc = Nokogiri::HTML(open('http://careerbuilder.vn')) doc = Nokogiri::HTML(open(@domain))
import_area
import_category(doc) import_category(doc)
import_city(doc) import_city(doc)
...@@ -83,12 +82,15 @@ class Careerbuilder ...@@ -83,12 +82,15 @@ class Careerbuilder
def detail(doc, _link) def detail(doc, _link)
# Company Information # Company Information
company = Company.new
company_name = doc.xpath("//div[@class='tit_company']").text.strip # Company name company_name = doc.xpath("//div[@class='tit_company']").text.strip # Company name
company_address = doc.xpath("//div[@class='box1Detail']/p[@class='TitleDetailNew']/label[@itemprop='address']/label[@itemprop='addressLocality']").text.strip # Company Address
company_description = doc.xpath("//div[@class='desc_company content_fck']").text.strip # Company description
company = Company.find_or_create_by(name: company_name)
company.name = company_name company.name = company_name
company.address = doc.xpath("//div[@class='box1Detail']/p[@class='TitleDetailNew']/label[@itemprop='address']/label[@itemprop='addressLocality']").text.strip # Company Address company.address = company_address
company.description = doc.xpath("//div[@class='desc_company content_fck']").text.strip # Company description company.description = company_description
company.save if Company.where(name: company_name).blank? company.save
# Job Information # Job Information
job_name = doc.xpath("//div[@class='LeftJobCB']/div[@class='top-job']/div[@class='top-job-info']/h1").text.strip # Job name job_name = doc.xpath("//div[@class='LeftJobCB']/div[@class='top-job']/div[@class='top-job-info']/h1").text.strip # Job name
...@@ -109,38 +111,26 @@ class Careerbuilder ...@@ -109,38 +111,26 @@ class Careerbuilder
city = City.find_by_name(job_location) city = City.find_by_name(job_location)
job = Job.new job = Job.find_or_create_by(name: job_name, city: city, company: company)
job.name = job_name
job.description = job_description job.description = job_description
job.salary = job_salary job.salary = job_salary
job.city = city
job.company = company
job.level = job_level job.level = job_level
job.experience = job_experience job.experience = job_experience
job.status = 0 job.status = 0
job.expiry_date = job_expiry_date.to_datetime job.expiry_date = job_expiry_date.to_datetime
job.save if Job.where(name: job_name, city: city, company: company).blank? job.save
job_category.split(',').each do |category| job_category.split(',').each do |category|
category = Category.find_by_name(category) category = Category.find_by_name(category)
JobCategory.new(job: job, category: category).save if JobCategory.where(job: job, category: category).blank? JobCategory.find_or_create_by(job: job, category: category)
end
end end
def import_area
Area.new(name: 'Viet Nam').save if Area.where(name: 'Viet Nam').blank?
Area.new(name: 'International').save if Area.where(name: 'International').blank?
rescue StandardError => e
logger.error("[method: ] #{import_category}")
logger.error(e.message)
logger.error(e.backtrace)
end end
def import_category(doc) def import_category(doc)
categories = doc.xpath("//div[@class='s-home2']/div[@id='NewSearchJob3']/form/div[@class='search-horizontal']/div[@class='ui-widget box_multiSelect_industry']/select/option") categories = doc.xpath("//div[@class='s-home2']/div[@id='NewSearchJob3']/form/div[@class='search-horizontal']/div[@class='ui-widget box_multiSelect_industry']/select/option")
categories = categories.slice(1..categories.size - 2) categories = categories.drop(1)
categories.each do |category| categories.each do |category|
Category.new(name: category.text.strip).save if Category.where(name: category.text.strip).blank? Category.find_or_create_by(name: category.text.strip)
end end
rescue StandardError => e rescue StandardError => e
logger.error("[method: ] #{import_category}") logger.error("[method: ] #{import_category}")
...@@ -150,10 +140,10 @@ class Careerbuilder ...@@ -150,10 +140,10 @@ class Careerbuilder
def import_city(doc) def import_city(doc)
cities = doc.xpath("//div[@class='s-home2']//div[@id='NewSearchJob3']/form/div[@class='search-horizontal']/div[@class='ui-widget box_multiSelect_location']/select/option").drop(1) cities = doc.xpath("//div[@class='s-home2']//div[@id='NewSearchJob3']/form/div[@class='search-horizontal']/div[@class='ui-widget box_multiSelect_location']/select/option").drop(1)
area_id = 1 area = Area.find_by_name('Viet Nam')
cities.each do |city| cities.each do |city|
area_id = 2 if city.text == 'Angola' area = Area.find_by_name('International') if city.text == 'Angola'
City.new(name: city.text.strip, area: Area.find(area_id)).save if City.where(name: city.text.strip).blank? City.find_or_create_by(name: city.text.strip, area: area)
end end
rescue StandardError => e rescue StandardError => e
logger.error("[method: ] #{import_city}") logger.error("[method: ] #{import_city}")
......
lib/import/csv_reader.rb 0 → 100644
require 'thread'
require 'logger'
require 'csv'
class Import::CSVReader
attr_reader :logger
def initialize(file)
@file = file
@logger = Logger.new("#{Rails.root}/log/csv_reader.log")
end
def import
@logger.info('Start read data')
puts '=======Start read data======='
csv_text = File.read(@file)
csv = CSV.parse(csv_text, headers: :true)
csv.each do |row|
begin
# Job type information
job_type = import_job_type(row)
# Contact information
contact = import_contact(row)
# Company information
company = import_company(row)
# Category information
category = import_category(row)
# City information
city = import_city(row)
# Job information
job = import_job(row, city, job_type, contact, company)
# Job Category Information
import_job_category(job, category)
rescue StandardError => e
logger.error(e.message)
logger.error(e.backtrace)
next
end
end
puts '=======End read data======='
@logger.info('End read data')
end
def import_job_type(row)
job_type_name = row['type'].strip unless row['type'].nil?
job_type = JobType.find_or_create_by(name: job_type_name)
job_type
end
def import_contact(row)
contact_name = row['contact name'].strip unless row['contact name'].nil?
contact_email = row['contact email'].strip unless row['contact email'].nil?
contact_phone = row['contact phone'].strip unless row['contact phone'].nil?
contact = Contact.find_or_create_by(email: contact_email)
contact.email = contact_email
contact.name = contact_name
contact.phone = contact_phone
contact.save
contact
end
def import_company(row)
company_address = row['company address'].strip unless row['company address'].nil?
company_district = row['company district'].strip unless row['company district'].nil?
company_name = row['company name'].strip unless row['company name'].nil?
company_province = row['company province'].strip unless row['company province'].nil?
company = Company.find_or_create_by(name: company_name)
company.address = company_address
company.district = company_district
company.name = company_name
company.province = company_province
company.save
company
end
def import_category(row)
category_name = row['category'].strip unless row['category'].nil?
category = Category.find_or_create_by(name: category_name)
category.name = category_name
category.save
category
end
def import_city(row)
city_name = row['work place'].strip unless row['work place'].nil?
city_name = city_name.tr('""', '').tr('[]', '') # Remove '["text"]' -> 'text'
city = City.find_or_create_by(name: city_name)
city.name = city_name
city.area = Area.find_by_name('Viet Nam')
city.save
city
end
def import_job(row, city, job_type, contact, company)
job_benefit = row['benefit'].strip unless row['benefit'].nil?
job_description = row['description'].strip unless row['description'].nil?
job_level = row['level'].strip unless row['level'].nil?
job_name = row['name'].strip unless row['name'].nil?
job_requirement = row['requirement'].strip unless row['requirement'].nil?
job_salary = row['salary'].strip unless row['salary'].nil?
job = Job.find_or_create_by(name: job_name, city: city, company: company)
job.benefit = job_benefit
job.description = job_description
job.level = job_level
job.name = job_name
job.requirement = job_requirement
job.salary = job_salary
job.city = city
job.job_type = job_type
job.contact = contact
job.company = company
job.save
job
end
def import_job_category(job, category)
job_category = JobCategory.find_or_create_by(job: job, category: category)
job_category.job = job
job_category.category = category
job_category.save
end
end
lib/tasks/crawler.rake
namespace :crawler do namespace :crawler do
desc 'client crawler' desc 'client crawler'
task load: :environment do task load: :environment do
require "#{Rails.root}/lib/tasks/careerbuilder"
thread_count = ENV['THREAD_COUNT'] || 1 thread_count = ENV['THREAD_COUNT'] || 1
Careerbuilder.new('http://careerbuilder.vn', thread_count.to_i).crawl Crawler::Careerbuilder.new('http://careerbuilder.vn', thread_count.to_i).crawl
end end
end end
lib/tasks/csv_reader.rb 0 → 100644
require 'thread'
require 'logger'
require 'csv'
class CSVReader
attr_reader :thread_count, :logger
def initialize(file, thread_count)
@file = file
@thread_count = thread_count
@mutex = Mutex.new
@logger = Logger.new("#{Rails.root}/log/csv_reader.log")
end
def import
@logger.info('Start read data')
workers = (0...thread_count).map do
Thread.new do
begin
begin
# Area information
Area.new(name: 'Viet Nam').save if Area.where(name: 'Viet Nam').blank?
Area.new(name: 'International').save if Area.where(name: 'International').blank?
csv_text = File.read(@file)
csv = CSV.parse(csv_text, headers: :true)
csv.each do |row|
# Job type information
job_type_name = row['type'].strip unless row['type'].nil?
job_type = JobType.new
job_type.name = job_type_name
job_type.save if JobType.find_by_name(job_type_name).blank?
# Contact information
contact_name = row['contact name'].strip unless row['contact name'].nil?
contact_email = row['contact email'].strip unless row['contact email'].nil?
contact_phone = row['contact phone'].strip unless row['contact phone'].nil?
contact = Contact.new
contact.email = contact_email
contact.name = contact_name
contact.phone = contact_phone
contact.save if Contact.find_by_email(contact_email).blank?
# Company information
company_address = row['company address'].strip unless row['company address'].nil?
company_district = row['company district'].strip unless row['company district'].nil?
company_name = row['company name'].strip unless row['company name'].nil?
company_province = row['company province'].strip unless row['company province'].nil?
company = Company.new
company.address = company_address
company.district = company_district
company.name = company_name
company.province = company_province
company.save if Company.find_by_name(company_name).blank?
# Category information
category_name = row['category'].strip unless row['category'].nil?
category = Category.new
category.name = category_name
category.save if Category.find_by_name(category_name).blank?
# City information
city_name = row['work place'].strip unless row['work place'].nil?
city = City.new
city.name = city_name
city.area = Area.find(1)
city.save if City.find_by_name(city_name).blank?
# Job information
job_benefit = row['benefit'].strip unless row['benefit'].nil?
job_description = row['description'].strip unless row['description'].nil?
job_level = row['level'].strip unless row['level'].nil?
job_name = row['name'].strip unless row['name'].nil?
job_requirement = row['requirement'].strip unless row['requirement'].nil?
job_salary = row['salary'].strip unless row['salary'].nil?
job = Job.new
job.benefit = job_benefit
job.description = job_description
job.level = job_level
job.name = job_name
job.requirement = job_requirement
job.salary = job_salary
job.city = city
job.job_type = job_type
job.contact = contact
job.company = company
job.save if Job.where(name: job_name, city: city, company: company).blank?
# Job Category Information
job_category = JobCategory.new
job_category.job = job
job_category.category = category
job_category.save if JobCategory.where(job: job, category: category).blank?
end
rescue StandardError => e
logger.error(e.message)
logger.error(e.backtrace)
end
puts '=======Thread End======='
rescue ThreadError
end
end
end
workers.map(&:join)
@logger.info('End read data')
end
end
lib/tasks/import_csv.rake 0 → 100644
require 'rubygems'
namespace :import do
desc 'Import CSV'
task csv: :environment do
path_zip = "#{Rails.root}/lib/tasks/jobs.zip"
Utils::Download.new('192.168.1.156', 'training', 'training', path_zip, '*.zip').download_ftp
path_csv = "#{Rails.root}/lib/tasks/jobs.csv"
Zip::File.open(path_zip) do |zipfile|
zipfile.each do |entry|
entry.extract(path_csv)
end
end
Import::CSVReader.new(path_csv).import
File.delete(path_zip)
File.delete(path_csv)
end
end
lib/utils/download.rb 0 → 100644
require 'net/ftp'
require 'zip'
class Utils::Download
attr_reader :url, :user, :password, :path, :file_type
def initialize(url, user, password, path, file_type)
@url = url
@user = user
@password = password
@path = path
@file_type = file_type
end
def download_ftp
ftp = Net::FTP.new
ftp.connect(@url)
ftp.login(@user, @password)
ftp.passive = true
files = ftp.nlst(@file_type)
files.each do |file_name|
ftp.getbinaryfile(file_name, @path)
end
ftp.close
end
end
test/controllers/sessions_controller_test.rb 0 → 100644
require 'test_helper'
class SessionsControllerTest < ActionDispatch::IntegrationTest
test "should get new" do
get sessions_new_url
assert_response :success
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment