Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
V
veNJOB
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Thanh Hung Pham
veNJOB
Commits
105faa0e
Commit
105faa0e
authored
Jul 06, 2017
by
Thanh Hung Pham
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fox code and remove thread
parent
7d3f85c2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
122 additions
and
93 deletions
+122
-93
lib/import/csv_reader.rb
+121
-91
lib/tasks/import_csv.rake
+1
-2
No files found.
lib/import/csv_reader.rb
View file @
105faa0e
...
...
@@ -3,106 +3,136 @@ require 'logger'
require
'csv'
class
Import
::
CSVReader
attr_reader
:
thread_count
,
:
logger
attr_reader
:logger
def
initialize
(
file
,
thread_count
)
def
initialize
(
file
)
@file
=
file
@thread_count
=
thread_count
@mutex
=
Mutex
.
new
@logger
=
Logger
.
new
(
"
#{
Rails
.
root
}
/log/csv_reader.log"
)
end
def
import
@logger
.
info
(
'Start read data'
)
workers
=
(
0
...
thread_count
).
map
do
Thread
.
new
do
begin
begin
csv_text
=
File
.
read
(
@file
)
csv
=
CSV
.
parse
(
csv_text
,
headers: :true
)
csv
.
each
do
|
row
|
# Job type information
job_type_name
=
row
[
'type'
].
strip
unless
row
[
'type'
].
nil?
job_type
=
JobType
.
find_or_create_by
(
name:
job_type_name
)
# Contact information
contact_name
=
row
[
'contact name'
].
strip
unless
row
[
'contact name'
].
nil?
contact_email
=
row
[
'contact email'
].
strip
unless
row
[
'contact email'
].
nil?
contact_phone
=
row
[
'contact phone'
].
strip
unless
row
[
'contact phone'
].
nil?
contact
=
Contact
.
find_or_create_by
(
email:
contact_email
)
contact
.
email
=
contact_email
contact
.
name
=
contact_name
contact
.
phone
=
contact_phone
contact
.
save
# Company information
company_address
=
row
[
'company address'
].
strip
unless
row
[
'company address'
].
nil?
company_district
=
row
[
'company district'
].
strip
unless
row
[
'company district'
].
nil?
company_name
=
row
[
'company name'
].
strip
unless
row
[
'company name'
].
nil?
company_province
=
row
[
'company province'
].
strip
unless
row
[
'company province'
].
nil?
company
=
Company
.
find_or_create_by
(
name:
company_name
)
company
.
address
=
company_address
company
.
district
=
company_district
company
.
name
=
company_name
company
.
province
=
company_province
company
.
save
# Category information
category_name
=
row
[
'category'
].
strip
unless
row
[
'category'
].
nil?
category
=
Category
.
find_or_create_by
(
name:
category_name
)
category
.
name
=
category_name
category
.
save
# City information
city_name
=
row
[
'work place'
].
strip
unless
row
[
'work place'
].
nil?
city_name
=
city_name
.
tr
(
'""'
,
''
).
tr
(
'[]'
,
''
)
# Remove '["text"]' -> 'text'
city
=
City
.
find_or_create_by
(
name:
city_name
)
city
.
name
=
city_name
city
.
area
=
Area
.
find_by_name
(
'Viet Nam'
)
city
.
save
# Job information
job_benefit
=
row
[
'benefit'
].
strip
unless
row
[
'benefit'
].
nil?
job_description
=
row
[
'description'
].
strip
unless
row
[
'description'
].
nil?
job_level
=
row
[
'level'
].
strip
unless
row
[
'level'
].
nil?
job_name
=
row
[
'name'
].
strip
unless
row
[
'name'
].
nil?
job_requirement
=
row
[
'requirement'
].
strip
unless
row
[
'requirement'
].
nil?
job_salary
=
row
[
'salary'
].
strip
unless
row
[
'salary'
].
nil?
job
=
Job
.
find_or_create_by
(
name:
job_name
,
city:
city
,
company:
company
)
job
.
benefit
=
job_benefit
job
.
description
=
job_description
job
.
level
=
job_level
job
.
name
=
job_name
job
.
requirement
=
job_requirement
job
.
salary
=
job_salary
job
.
city
=
city
job
.
job_type
=
job_type
job
.
contact
=
contact
job
.
company
=
company
job
.
save
# Job Category Information
job_category
=
JobCategory
.
find_or_create_by
(
job:
job
,
category:
category
)
job_category
.
job
=
job
job_category
.
category
=
category
job_category
.
save
end
rescue
StandardError
=>
e
logger
.
error
(
e
.
message
)
logger
.
error
(
e
.
backtrace
)
end
puts
'=======Thread End======='
rescue
ThreadError
end
puts
'=======Start read data======='
begin
csv_text
=
File
.
read
(
@file
)
csv
=
CSV
.
parse
(
csv_text
,
headers: :true
)
csv
.
each
do
|
row
|
# Job type information
job_type
=
import_job_type
(
row
)
# Contact information
contact
=
import_contact
(
row
)
# Company information
company
=
import_company
(
row
)
# Category information
category
=
import_category
(
row
)
# City information
city
=
import_city
(
row
)
# Job information
job
=
import_job
(
row
,
city
,
job_type
,
contact
,
company
)
# Job Category Information
import_job_category
(
job
,
category
)
end
rescue
StandardError
=>
e
logger
.
error
(
e
.
message
)
logger
.
error
(
e
.
backtrace
)
end
workers
.
map
(
&
:join
)
puts
'=======End read data======='
@logger
.
info
(
'End read data'
)
end
def
import_job_type
(
row
)
job_type_name
=
row
[
'type'
].
strip
unless
row
[
'type'
].
nil?
job_type
=
JobType
.
find_or_create_by
(
name:
job_type_name
)
job_type
end
def
import_contact
(
row
)
contact_name
=
row
[
'contact name'
].
strip
unless
row
[
'contact name'
].
nil?
contact_email
=
row
[
'contact email'
].
strip
unless
row
[
'contact email'
].
nil?
contact_phone
=
row
[
'contact phone'
].
strip
unless
row
[
'contact phone'
].
nil?
contact
=
Contact
.
find_or_create_by
(
email:
contact_email
)
contact
.
email
=
contact_email
contact
.
name
=
contact_name
contact
.
phone
=
contact_phone
contact
.
save
contact
end
def
import_company
(
row
)
company_address
=
row
[
'company address'
].
strip
unless
row
[
'company address'
].
nil?
company_district
=
row
[
'company district'
].
strip
unless
row
[
'company district'
].
nil?
company_name
=
row
[
'company name'
].
strip
unless
row
[
'company name'
].
nil?
company_province
=
row
[
'company province'
].
strip
unless
row
[
'company province'
].
nil?
company
=
Company
.
find_or_create_by
(
name:
company_name
)
company
.
address
=
company_address
company
.
district
=
company_district
company
.
name
=
company_name
company
.
province
=
company_province
company
.
save
company
end
def
import_category
(
row
)
category_name
=
row
[
'category'
].
strip
unless
row
[
'category'
].
nil?
category
=
Category
.
find_or_create_by
(
name:
category_name
)
category
.
name
=
category_name
category
.
save
category
end
def
import_city
(
row
)
city_name
=
row
[
'work place'
].
strip
unless
row
[
'work place'
].
nil?
city_name
=
city_name
.
tr
(
'""'
,
''
).
tr
(
'[]'
,
''
)
# Remove '["text"]' -> 'text'
city
=
City
.
find_or_create_by
(
name:
city_name
)
city
.
name
=
city_name
city
.
area
=
Area
.
find_by_name
(
'Viet Nam'
)
city
.
save
city
end
def
import_job
(
row
,
city
,
job_type
,
contact
,
company
)
job_benefit
=
row
[
'benefit'
].
strip
unless
row
[
'benefit'
].
nil?
job_description
=
row
[
'description'
].
strip
unless
row
[
'description'
].
nil?
job_level
=
row
[
'level'
].
strip
unless
row
[
'level'
].
nil?
job_name
=
row
[
'name'
].
strip
unless
row
[
'name'
].
nil?
job_requirement
=
row
[
'requirement'
].
strip
unless
row
[
'requirement'
].
nil?
job_salary
=
row
[
'salary'
].
strip
unless
row
[
'salary'
].
nil?
job
=
Job
.
find_or_create_by
(
name:
job_name
,
city:
city
,
company:
company
)
job
.
benefit
=
job_benefit
job
.
description
=
job_description
job
.
level
=
job_level
job
.
name
=
job_name
job
.
requirement
=
job_requirement
job
.
salary
=
job_salary
job
.
city
=
city
job
.
job_type
=
job_type
job
.
contact
=
contact
job
.
company
=
company
job
.
save
job
end
def
import_job_category
(
job
,
category
)
job_category
=
JobCategory
.
find_or_create_by
(
job:
job
,
category:
category
)
job_category
.
job
=
job
job_category
.
category
=
category
job_category
.
save
end
end
lib/tasks/import_csv.rake
View file @
105faa0e
...
...
@@ -3,7 +3,6 @@ require 'rubygems'
namespace
:import
do
desc
'Import CSV'
task
csv: :environment
do
thread_count
=
ENV
[
'THREAD_COUNT'
]
||
1
path_zip
=
"
#{
Rails
.
root
}
/lib/tasks/jobs.zip"
Utils
::
Download
.
new
(
'192.168.1.156'
,
'training'
,
'training'
,
path_zip
,
'*.zip'
).
download_ftp
...
...
@@ -15,7 +14,7 @@ namespace :import do
end
end
Import
::
CSVReader
.
new
(
path_csv
,
thread_count
).
import
Import
::
CSVReader
.
new
(
path_csv
).
import
File
.
delete
(
path_zip
)
File
.
delete
(
path_csv
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment