Commit 931465fe by Tô Ngọc Ánh

fix no directory, refactor code in csv.rb

parent c99884c7
Pipeline #726 canceled with stages
in 0 seconds
......@@ -9,10 +9,8 @@ class CsvImport
end
def import_job(direction)
index = 0
CSV.foreach("#{direction}/jobs.csv", headers: true) do |row|
index += 1
next if integer?(row['category'])
CSV.foreach("#{direction}/jobs.csv", headers: true).with_index(2) do |row, index|
next if row['category'].blank? || row['category'].match(/^[0-9]+$/).present?
title = row['name'].strip
company = Company.find_or_create_by(name: row['company name']) do |c|
......@@ -37,15 +35,9 @@ class CsvImport
job.description = description
end
puts title
end
rescue StandardError => e
puts e
@logger.error "Job #{index}: #{e.message}"
end
private
def integer?(str)
str.to_i.to_s == str
end
end
......@@ -6,15 +6,16 @@ namespace :import_data do
logger ||= Logger.new('./log/import_data.log')
desc 'crawl industries locations jobs'
task :crawler, %i[page link] => [:environment] do |_, args|
args.with_defaults(page: 1, link: 'https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html')
task :crawler, %i[page_number link] => [:environment] do |_, args|
args.with_defaults(page_number: 1, link: 'https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html')
crawler = Crawler.new(logger)
crawler.crawl_data(args[:page].to_i, args[:link])
crawler.crawl_data(args[:page_number].to_i, args[:link])
end
desc 'Download csv file from FTP and import'
task csv: :environment do
destination_dir = './lib/data'
Dir.mkdir destination_dir unless File.exists?(destination_dir)
ftp = Ftp.new('192.168.1.156', 'training', 'training')
ftp.download_file('jobs.zip', destination_dir)
ftp.close
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment