create import csv function
Chức năng: Tải file CSV từ FTP Server vể, tự giải nén, parse data và import data vào db
-
-
lib/service/ftp.rb 0 → 100644
1 # frozen_string_literal: true 2 3 require 'service/unzip' 4 require 'net/ftp' 5 require 'csv' 6 7 # Access Ftp Server get file & extract file 8 class FtpSever 9 CONTENT_SERVER_DOMAIN_NAME = '192.168.1.156' 10 CONTENT_SERVER_USER_NAME = 'training' 11 CONTENT_SERVER_USER_PASSWORD = 'training' 12 def self.donwload_csv -
MasterEdited
Có thể dùng cách khác để khai báo class methods:
Đặt các method giữa:
class << selfclass A class << self def method_a; end def method_b; end end def method_a; end end -
Master
Dạ anh
-
-
-
lib/service/ftp.rb 0 → 100644
5 require 'csv' 6 7 # Access Ftp Server get file & extract file 8 class FtpSever 9 CONTENT_SERVER_DOMAIN_NAME = '192.168.1.156' 10 CONTENT_SERVER_USER_NAME = 'training' 11 CONTENT_SERVER_USER_PASSWORD = 'training' 12 def self.donwload_csv 13 Net::FTP.open(CONTENT_SERVER_DOMAIN_NAME, CONTENT_SERVER_USER_NAME, CONTENT_SERVER_USER_PASSWORD) do |ftp| 14 begin 15 ftp.getbinaryfile('jobs.zip') 16 extract_zip('./jobs.zip', 'lib/csv') 17 File.delete('./jobs.zip') if File.exist?('./jobs.zip') 18 puts 'Extract file done' 19 rescue StandardError => e 20 puts "#{e} File not found" -
MasterEdited
Nên ghi vào log để tiện theo dõi vì các rake task này sẽ được chạy bằng crontab
Sử dụng
putschỉ in ra màn hình console, khó theo dõi khi chạy bằng crontab -
Master
Dạ anh
-
-
-
lib/service/import_data.rb 0 → 100644
1 # frozen_string_literal: true 2 3 require 'service/ftp' 4 5 # Import data from CSV 6 class ImportDataCSV -
MasterEdited by Hoang Phuc Do
Tên class và tên file phải tương ứng với nhau
Tên file đặt theo định dạng snake_case
Tên class đặt theo định dạng CamelCase
import_data.rb=>class ImportData -
Master
Dạ
-
-
-
lib/service/unzip.rb 0 → 100644
1 # frozen_string_literal: true 2 3 require 'zip' 4 5 # Description/Explanation of Person class 6 def extract_zip(file, destination) -
Master
Đặt method này bên trong một module hoặc class
-
-
-
lib/service/ftp.rb 0 → 100644
4 require 'net/ftp' 5 require 'csv' 6 7 # Access Ftp Server get file & extract file 8 class FtpSever 9 CONTENT_SERVER_DOMAIN_NAME = '192.168.1.156' 10 CONTENT_SERVER_USER_NAME = 'training' 11 CONTENT_SERVER_USER_PASSWORD = 'training' 12 def self.donwload_csv 13 Net::FTP.open(CONTENT_SERVER_DOMAIN_NAME, CONTENT_SERVER_USER_NAME, CONTENT_SERVER_USER_PASSWORD) do |ftp| 14 begin 15 ftp.getbinaryfile('jobs.zip') 16 extract_zip('./jobs.zip', 'lib/csv') 17 File.delete('./jobs.zip') if File.exist?('./jobs.zip') 18 puts 'Extract file done' 19 rescue StandardError => e -
Master
StandardErrorlà lỗi tổng quát, không chỉ là File not found -
-
-
lib/service/import_data.rb 0 → 100644
43 def parse_csv_companies(data) 44 puts 'Import data companies . . .' 45 data['company name'].each_with_index do |name, index| 46 Company.find_or_create_by(name: name.strip) do |company| 47 company.address = data['company address'][index] 48 company.short_description = data['benefit'][index] 49 end 50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def parse_csv_jobs(data) 57 data['name'].each_with_index do |name, index| 58 desc = (data['requirement'][index]).to_s << (data['description'][index]).to_s -
Master
Nối chuỗi dùng string interpolation
"#{data['requirement'][index]} #{data['description'][index]}" -
-
-
lib/service/import_data.rb 0 → 100644
44 puts 'Import data companies . . .' 45 data['company name'].each_with_index do |name, index| 46 Company.find_or_create_by(name: name.strip) do |company| 47 company.address = data['company address'][index] 48 company.short_description = data['benefit'][index] 49 end 50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def parse_csv_jobs(data) 57 data['name'].each_with_index do |name, index| 58 desc = (data['requirement'][index]).to_s << (data['description'][index]).to_s 59 id_company = Company.find_by name: data['company name'][index].to_s.strip -
Master
Đặt tên variable không chính xác,
Company.find_bytrả về một ActiveRecord objectCó thể đặt là
company -
-
-
lib/service/import_data.rb 0 → 100644
45 data['company name'].each_with_index do |name, index| 46 Company.find_or_create_by(name: name.strip) do |company| 47 company.address = data['company address'][index] 48 company.short_description = data['benefit'][index] 49 end 50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def parse_csv_jobs(data) 57 data['name'].each_with_index do |name, index| 58 desc = (data['requirement'][index]).to_s << (data['description'][index]).to_s 59 id_company = Company.find_by name: data['company name'][index].to_s.strip 60 id_company = id_company.blank? ? COMPANY_SECURITY : id_company.id -
Master
=>
company_id -
-
-
lib/service/import_data.rb 0 → 100644
47 company.address = data['company address'][index] 48 company.short_description = data['benefit'][index] 49 end 50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def parse_csv_jobs(data) 57 data['name'].each_with_index do |name, index| 58 desc = (data['requirement'][index]).to_s << (data['description'][index]).to_s 59 id_company = Company.find_by name: data['company name'][index].to_s.strip 60 id_company = id_company.blank? ? COMPANY_SECURITY : id_company.id 61 begin 62 id_job = Job.create!(name: name, -
Master
id_job=>job -
-
-
lib/service/import_data.rb 0 → 100644
50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def parse_csv_jobs(data) 57 data['name'].each_with_index do |name, index| 58 desc = (data['requirement'][index]).to_s << (data['description'][index]).to_s 59 id_company = Company.find_by name: data['company name'][index].to_s.strip 60 id_company = id_company.blank? ? COMPANY_SECURITY : id_company.id 61 begin 62 id_job = Job.create!(name: name, 63 company_id: id_company, 64 level: data['level'][index], 65 experience: '', -
Master
Nếu không có giá trị có thể bỏ
experiencekhỏi tham số được không ? -
Master
Dạ được anh, để e sửa lại
-
-
-
lib/service/import_data.rb 0 → 100644
53 end 54 end 55 56 def parse_csv_jobs(data) 57 data['name'].each_with_index do |name, index| 58 desc = (data['requirement'][index]).to_s << (data['description'][index]).to_s 59 id_company = Company.find_by name: data['company name'][index].to_s.strip 60 id_company = id_company.blank? ? COMPANY_SECURITY : id_company.id 61 begin 62 id_job = Job.create!(name: name, 63 company_id: id_company, 64 level: data['level'][index], 65 experience: '', 66 salary: data['salary'][index], 67 create_date: Time.now, 68 expiration_date: '', -
Master
Nếu không có giá trị có thể bỏ
expiration_datekhỏi tham số được không ? -
-
-
lib/service/import_data.rb 0 → 100644
5 # Import data from CSV 6 class ImportDataCSV 7 DOMESTIC = 1 8 COMPANY_SECURITY = 1 9 10 def import_data 11 csv_data = FtpSever.data_csv 12 parse_csv_industries(csv_data) 13 parse_csv_cities(csv_data) 14 parse_csv_companies(csv_data) 15 parse_csv_jobs(csv_data) 16 end 17 18 private 19 20 def parse_csv_industries(data) -
Master
parse_csv_industries(data)=>import_industries_from(csv) -
-
-
-
-
lib/service/import_data.rb 0 → 100644
41 end 42 43 def parse_csv_companies(data) 44 puts 'Import data companies . . .' 45 data['company name'].each_with_index do |name, index| 46 Company.find_or_create_by(name: name.strip) do |company| 47 company.address = data['company address'][index] 48 company.short_description = data['benefit'][index] 49 end 50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def parse_csv_jobs(data) -
Master
parse_csv_jobs(data)=>import_jobs_from(csv) -
-
-
-
-
lib/service/import_data.rb 0 → 100644
1 # frozen_string_literal: true 2 3 require 'service/ftp' 4 5 # Import data from CSV 6 class ImportDataCSV 7 DOMESTIC = 1 -
MasterEdited by Hoang Phuc Do
Giá trị này thuộc phần xử lý của model City
=> Chuyển giá trị này vào
app/models/city.rbTham khảo cách sử dụng
enum trong Rails -
-
-
lib/service/ftp.rb 0 → 100644
1 # frozen_string_literal: true 2 3 require 'service/unzip' 4 require 'net/ftp' 5 require 'csv' 6 7 # Access Ftp Server get file & extract file 8 class FtpSever 9 CONTENT_SERVER_DOMAIN_NAME = '192.168.1.156' 10 CONTENT_SERVER_USER_NAME = 'training' 11 CONTENT_SERVER_USER_PASSWORD = 'training' 12 def self.donwload_csv 13 Net::FTP.open(CONTENT_SERVER_DOMAIN_NAME, CONTENT_SERVER_USER_NAME, CONTENT_SERVER_USER_PASSWORD) do |ftp| 14 begin 15 ftp.getbinaryfile('jobs.zip') 16 extract_zip('./jobs.zip', 'lib/csv') -
-
Master
Dạ anh
-
-
-
-
lib/service/import_data.rb 0 → 100644
70 puts index 71 rescue StandardError => e 72 puts e 73 end 74 end 75 end 76 77 def make_foreign_cities_table(data, id_job) 78 data = data.to_s.delete('[]\"') 79 city = City.find_or_create_by(name: data.strip) { |record| record.area = DOMESTIC } 80 city_id = city.id 81 CityJob.create!(job_id: id_job, city_id: city_id) 82 end 83 84 def make_foreign_industries_table(data, id_job) 85 data = data.to_s.gsub(',', '/').gsub('/', ' / ') -
Master
Giá trị này có vẻ được dùng để làm industry name. Tên biến
datacó nghĩa khá rộng và không phản ảnh được mục đích sử dụngdata=>industry_nameNgoài ra, hạn chế đặt tên biến trùng với tham số của method.
-
-
-
lib/service/import_data.rb 0 → 100644
73 end 74 end 75 end 76 77 def make_foreign_cities_table(data, id_job) 78 data = data.to_s.delete('[]\"') 79 city = City.find_or_create_by(name: data.strip) { |record| record.area = DOMESTIC } 80 city_id = city.id 81 CityJob.create!(job_id: id_job, city_id: city_id) 82 end 83 84 def make_foreign_industries_table(data, id_job) 85 data = data.to_s.gsub(',', '/').gsub('/', ' / ') 86 industry = Industry.find_or_create_by(name: data.strip) 87 industry_id = industry.id 88 IndustryJob.create!(industry_id: industry_id, job_id: id_job) -
MasterEdited by Hoang Phuc Do
Sử dụng
create!sẽ raise lỗi khi không lưu được record xuống DB.Việc raise lỗi ở đây nhằm mục đích gì ?
-
-
-
-
-
lib/service/ftp.rb 0 → 100644
2 3 require 'service/unzip' 4 require 'net/ftp' 5 require 'logger' 6 require 'csv' 7 8 # Access Ftp Server get file & extract file 9 class FtpSever 10 CONTENT_SERVER_DOMAIN_NAME = '192.168.1.156' 11 CONTENT_SERVER_USER_NAME = 'training' 12 CONTENT_SERVER_USER_PASSWORD = 'training' 13 NAME_CSV = 'jobs.zip' 14 15 def data_csv 16 donwload_csv 17 CSV.parse(File.read('lib/csv/jobs.csv'), headers: true) -
Master
CSV.parse(File.read(Rails.root.join('lib', 'csv', 'jobs.csv')), headers: true) -
-
-
lib/service/ftp.rb 0 → 100644
11 CONTENT_SERVER_USER_NAME = 'training' 12 CONTENT_SERVER_USER_PASSWORD = 'training' 13 NAME_CSV = 'jobs.zip' 14 15 def data_csv 16 donwload_csv 17 CSV.parse(File.read('lib/csv/jobs.csv'), headers: true) 18 end 19 20 def logger 21 @logger ||= Logger.new(Rails.root.join('log', 'csv.log')) 22 end 23 24 private 25 26 def donwload_csv -
Master
donwload_csv=>download_csv -
-
-
lib/service/ftp.rb 0 → 100644
16 donwload_csv 17 CSV.parse(File.read('lib/csv/jobs.csv'), headers: true) 18 end 19 20 def logger 21 @logger ||= Logger.new(Rails.root.join('log', 'csv.log')) 22 end 23 24 private 25 26 def donwload_csv 27 Net::FTP.open(CONTENT_SERVER_DOMAIN_NAME, CONTENT_SERVER_USER_NAME, CONTENT_SERVER_USER_PASSWORD) do |ftp| 28 ftp.getbinaryfile(NAME_CSV) 29 Unzip.extract_zip(Rails.root.join(NAME_CSV), Rails.root.join('lib', 'csv')) 30 File.delete(Rails.root.join(NAME_CSV)) if File.exist?(Rails.root.join(NAME_CSV)) 31 logger.info 'Donwload & extract success' -
Master
logger.info 'Download & extract success' -
-
-
lib/service/ftp.rb 0 → 100644
15 def data_csv 16 donwload_csv 17 CSV.parse(File.read('lib/csv/jobs.csv'), headers: true) 18 end 19 20 def logger 21 @logger ||= Logger.new(Rails.root.join('log', 'csv.log')) 22 end 23 24 private 25 26 def donwload_csv 27 Net::FTP.open(CONTENT_SERVER_DOMAIN_NAME, CONTENT_SERVER_USER_NAME, CONTENT_SERVER_USER_PASSWORD) do |ftp| 28 ftp.getbinaryfile(NAME_CSV) 29 Unzip.extract_zip(Rails.root.join(NAME_CSV), Rails.root.join('lib', 'csv')) 30 File.delete(Rails.root.join(NAME_CSV)) if File.exist?(Rails.root.join(NAME_CSV)) -
Master
Rails.root.join(NAME_CSV)bị lặp lạiCó thể gán thành một biến hoặc một method
-
Master
Rails.root.join(NAME_CSV) e sẽ cho nó vào method, anh gợi ý cho e 1 cái tên method đi
-
Master
Có thể dựa vào các tiêu chí sau để đặt tên phù hợp:
- Kiểu dữ liệu trả về
- Ngữ cảnh sử dụng
Trong trường hợp này, có thể phân tích như sau:
- Giá trị là đường dẫn của file csv
- Chứa các dữ liệu dành cho job
=> Tên phù hợp có thể là
jobs_csv_path -
-
-
lib/service/import_data.rb 0 → 100644
14 import_companies_from(csv_data) 15 import_jobs_from(csv_data) 16 end 17 18 private 19 20 def import_industries_from(csv) 21 puts 'Import data industries . . .' 22 industries = [] 23 industries += csv['category'].map(&:strip) 24 industries.each do |val| 25 val.gsub!(',', '/') if val.include?(',') 26 val.gsub!('/', ' / ') 27 Industry.find_or_create_by(name: val) 28 end 29 puts 'Done parse csv industries' -
Master
Trong này cũng nên sử dụng logger
-
-
-
lib/service/import_data.rb 0 → 100644
66 create_date: Time.now, 67 description: desc) 68 make_foreign_cities_table(csv['work place'][index], job.id) 69 make_foreign_industries_table(csv['category'][index], job.id) 70 puts index 71 rescue StandardError => e 72 puts e 73 end 74 end 75 end 76 77 def make_foreign_cities_table(data, id_job) 78 data = data.to_s.delete('[]\"') 79 city = City.find_or_create_by(name: data.strip) { |record| record.area = DOMESTIC } 80 city_id = city.id 81 CityJob.create(job_id: id_job, city_id: city_id) -
Master
Ngoài cách gán
job_idtrực tiếp, có thể tạo dữ liệu cho association thông qua objectjobkhông ? -
-
-
lib/service/import_data.rb 0 → 100644
47 company.address = csv['company address'][index] 48 company.short_description = csv['benefit'][index] 49 end 50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def import_jobs_from(csv) 57 csv['name'].each_with_index do |name, index| 58 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 59 company = Company.find_by name: csv['company name'][index].to_s.strip 60 company_id = company.blank? ? COMPANY_SECURITY : company.id 61 begin 62 job = Job.create!(name: name, -
Master
Nếu ở đây việc tạo job bị fail thì sẽ ngừng luôn việc xử lý.
Có thể xử lý để tiếp tục thực hiện import tiếp các job khác không ?
-
-
-
lib/service/import_data.rb 0 → 100644
19 20 def import_industries_from(csv) 21 puts 'Import data industries . . .' 22 industries = [] 23 industries += csv['category'].map(&:strip) 24 industries.each do |val| 25 val.gsub!(',', '/') if val.include?(',') 26 val.gsub!('/', ' / ') 27 Industry.find_or_create_by(name: val) 28 end 29 puts 'Done parse csv industries' 30 end 31 32 def import_cities_from(csv) 33 puts 'Import data cities . . .' 34 cities = csv['work place'].uniq.select(&:present?) -
Master
Tên phù hợp hơn có thể là
city_names -
-
-
lib/service/import_data.rb 0 → 100644
51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def import_jobs_from(csv) 57 csv['name'].each_with_index do |name, index| 58 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 59 company = Company.find_by name: csv['company name'][index].to_s.strip 60 company_id = company.blank? ? COMPANY_SECURITY : company.id 61 begin 62 job = Job.create!(name: name, 63 company_id: company_id, 64 level: csv['level'][index], 65 salary: csv['salary'][index], 66 create_date: Time.now, -
Master
Time.nowvàTime.current, có sự khác biệt gì không ? -
Master
Time.current sẽ trả về Time.zone.now khi Time.zone hoặc config.time_zone được đặt Time.now trả về thời gian hiện tại
-
-
-
lib/service/import_data.rb 0 → 100644
53 end 54 end 55 56 def import_jobs_from(csv) 57 csv['name'].each_with_index do |name, index| 58 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 59 company = Company.find_by name: csv['company name'][index].to_s.strip 60 company_id = company.blank? ? COMPANY_SECURITY : company.id 61 begin 62 job = Job.create!(name: name, 63 company_id: company_id, 64 level: csv['level'][index], 65 salary: csv['salary'][index], 66 create_date: Time.now, 67 description: desc) 68 make_foreign_cities_table(csv['work place'][index], job.id) -
Master
Tên phù hợp có thể là
create_city_relation -
-
-
lib/service/import_data.rb 0 → 100644
54 end 55 56 def import_jobs_from(csv) 57 csv['name'].each_with_index do |name, index| 58 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 59 company = Company.find_by name: csv['company name'][index].to_s.strip 60 company_id = company.blank? ? COMPANY_SECURITY : company.id 61 begin 62 job = Job.create!(name: name, 63 company_id: company_id, 64 level: csv['level'][index], 65 salary: csv['salary'][index], 66 create_date: Time.now, 67 description: desc) 68 make_foreign_cities_table(csv['work place'][index], job.id) 69 make_foreign_industries_table(csv['category'][index], job.id) -
Master
Tên phù hợp có thể là
create_industry_relation -
-
-
lib/service/import_data.rb 0 → 100644
45 csv['company name'].each_with_index do |name, index| 46 Company.find_or_create_by(name: name.strip) do |company| 47 company.address = csv['company address'][index] 48 company.short_description = csv['benefit'][index] 49 end 50 puts index 51 rescue StandardError => e 52 puts e 53 end 54 end 55 56 def import_jobs_from(csv) 57 csv['name'].each_with_index do |name, index| 58 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 59 company = Company.find_by name: csv['company name'][index].to_s.strip 60 company_id = company.blank? ? COMPANY_SECURITY : company.id -
Master
Một số cách khác
company_id = company.try(:id) || COMPANY_SECURITYcompany_id = company&.id || COMPANY_SECURITYCó gì khác giữa
try(:id)và&.id -
-
-
-
lib/service/import_data.rb 0 → 100644
27 industries.each do |val| 28 val.gsub!(',', '/') if val.include?(',') 29 val.gsub!('/', ' / ') 30 Industry.find_or_create_by(name: val) 31 end 32 rescue StandardError => e 33 logger.error "Import_industries: #{e}" 34 end 35 36 def import_cities_from(csv) 37 city_names = csv['work place'].uniq.select(&:present?) 38 cities = city_names.map { |val| val.delete('[]\"') } 39 cities.each do |val| 40 next if val.blank? 41 42 City.find_or_create_by(name: val) { |city| city.area = DOMESTIC } -
Master
area của city luôn cố định là
DOMESTICVậy có thể chuyển thành như vậy không ?
City.find_or_create_by(name: val, area: DOMESTIC) -
-
-
lib/service/import_data.rb 0 → 100644
50 Company.find_or_create_by(name: name.strip) do |company| 51 company.address = csv['company address'][index] 52 company.short_description = csv['benefit'][index] 53 end 54 rescue StandardError => e 55 logger.error "Import_companies: #{e}" 56 end 57 end 58 59 def import_jobs_from(csv) 60 csv['name'].each_with_index do |name, index| 61 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 62 company = Company.find_by name: csv['company name'][index].to_s.strip 63 company_id = company.try(:id) || COMPANY_SECURITY 64 begin 65 job = Job.create(name: name, -
Master
Với cách khai báo như dưới đây thì có khác gì khác biệt so với cách trên về cách lưu dữ liệu vào DB
job = Job.new(name: name, ...) job.cities << city job.industries << industry job.save -
-
-
lib/service/import_data.rb 0 → 100644
57 end 58 59 def import_jobs_from(csv) 60 csv['name'].each_with_index do |name, index| 61 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 62 company = Company.find_by name: csv['company name'][index].to_s.strip 63 company_id = company.try(:id) || COMPANY_SECURITY 64 begin 65 job = Job.create(name: name, 66 company_id: company_id, 67 level: csv['level'][index], 68 salary: csv['salary'][index], 69 create_date: Time.now, 70 description: desc) 71 city_names = csv['work place'][index].to_s.delete('[]\"') 72 city = City.find_or_create_by(name: city_names.strip) { |record| record.area = DOMESTIC } -
Master
Có thể chuyển thành một method nếu dùng nhiều chỗ
-
-
-
lib/service/import_data.rb 0 → 100644
57 end 58 59 def import_jobs_from(csv) 60 csv['name'].each_with_index do |name, index| 61 desc = "#{csv['requirement'][index]} #{(csv['description'][index])}" 62 company = Company.find_by name: csv['company name'][index].to_s.strip 63 company_id = company.try(:id) || COMPANY_SECURITY 64 begin 65 job = Job.create(name: name, 66 company_id: company_id, 67 level: csv['level'][index], 68 salary: csv['salary'][index], 69 create_date: Time.now, 70 description: desc) 71 city_names = csv['work place'][index].to_s.delete('[]\"') 72 city = City.find_or_create_by(name: city_names.strip) { |record| record.area = DOMESTIC } -
Master
Có thể chuyển thành một method nếu dùng nhiều chỗ
-
-
-
-
-
-
-
added 31 commits
-
ba6efd34...c6b99ebd - 30 commits from branch
master - 1fb23a64 - Merge branch 'master' into 'csv'
Toggle commit list -
ba6efd34...c6b99ebd - 30 commits from branch
-
merged
Toggle commit list