Skip to content

V
ventura-ghr
Commit fbdb9b04 by Bui Minh Duc
Options

optimize github import

parent 29b46411
Showing with 282 additions and 117 deletions
Gemfile
...@@ -37,6 +37,7 @@ gem 'bootstrap-select-rails', '~> 1.6', '>= 1.6.3' ...@@ -37,6 +37,7 @@ gem 'bootstrap-select-rails', '~> 1.6', '>= 1.6.3'
gem 'bootstrap-table-rails', '~> 1.11' gem 'bootstrap-table-rails', '~> 1.11'
gem 'momentjs-rails', '>= 2.9.0' gem 'momentjs-rails', '>= 2.9.0'
gem 'bootstrap3-datetimepicker-rails', '~> 4.14.30' gem 'bootstrap3-datetimepicker-rails', '~> 4.14.30'
gem 'whenever', '~> 0.9.7'
group :development, :test do group :development, :test do
# Call 'byebug' anywhere in the code to stop execution and get a debugger console # Call 'byebug' anywhere in the code to stop execution and get a debugger console
......
Gemfile.lock
...@@ -52,6 +52,7 @@ GEM ...@@ -52,6 +52,7 @@ GEM
momentjs-rails (>= 2.8.1) momentjs-rails (>= 2.8.1)
builder (3.2.2) builder (3.2.2)
byebug (9.0.6) byebug (9.0.6)
chronic (0.10.2)
coffee-rails (4.2.1) coffee-rails (4.2.1)
coffee-script (>= 2.2.0) coffee-script (>= 2.2.0)
railties (>= 4.0.0, < 5.2.x) railties (>= 4.0.0, < 5.2.x)
...@@ -171,6 +172,8 @@ GEM ...@@ -171,6 +172,8 @@ GEM
websocket-driver (0.6.4) websocket-driver (0.6.4)
websocket-extensions (>= 0.1.0) websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.2) websocket-extensions (0.1.2)
whenever (0.9.7)
chronic (>= 0.6.3)
will_paginate (3.1.5) will_paginate (3.1.5)
PLATFORMS PLATFORMS
...@@ -198,6 +201,7 @@ DEPENDENCIES ...@@ -198,6 +201,7 @@ DEPENDENCIES
tzinfo-data tzinfo-data
uglifier (>= 1.3.0) uglifier (>= 1.3.0)
web-console web-console
whenever (~> 0.9.7)
will_paginate (~> 3.1, >= 3.1.5) will_paginate (~> 3.1, >= 3.1.5)
BUNDLED WITH BUNDLED WITH
......
app/assets/javascripts/application.js
...@@ -19,5 +19,6 @@ ...@@ -19,5 +19,6 @@
//= require bootstrap/alert //= require bootstrap/alert
//= require bootstrap/dropdown //= require bootstrap/dropdown
//= require bootstrap-table //= require bootstrap-table
//= require extensions/bootstrap-table-export.js
//= require moment //= require moment
//= require bootstrap-datetimepicker //= require bootstrap-datetimepicker
app/controllers/main_controller.rb
...@@ -2,26 +2,27 @@ class MainController < ApplicationController ...@@ -2,26 +2,27 @@ class MainController < ApplicationController
def index def index
@repos = @repos || Repository.all @repos = @repos || Repository.all
@selected_repos = Repository.where(selected: true) @selected_repos_name = Other.where(data_type: 0).map{ |other| other.data }
@selected_repos = Repository.where(name: @selected_repos_name)
@label_filter = ["ventura", "priority-ugent", "priority-high", "priority-low"] @label_filter = ["ventura", "priority-ugent", "priority-high", "priority-low"]
@col_label_names = [ @col_label_names = [
"discussion", "todo", "inprogress", "discussion", "todo", "inprogress",
"vnreview", "jpreview", "ready", "vnreview", "jpreview", "ready",
"done", "releasefailed", "pending", "ventura"] "done", "releasefailed", "pending", "ventura", "prioritylow", "priorityhigh", "priorityurgent"]
@selected_repos_name = []
@data_table = [] @data_table = []
@selected_repos.each do |repo| @selected_repos.each do |repo|
@selected_repos_name.append(repo.name)
row_table = Hash.new row_table = Hash.new
row_table[:name] = repo.name row_table[:name] = repo.name
row_table[:link] = repo.html_url row_table[:link] = repo.html_url
row_table[:having_data] = false
repo.issues.each do |issue| repo.issues.each do |issue|
issue.labels.each do |label| issue.labels.each do |label|
label_name = standardize_string(label[:name]) label_name = standardize_string(label[:name])
@col_label_names.each do |col_label| @col_label_names.each do |col_label|
if label_name == col_label if label_name == col_label
row_table[:having_data] = true
cell = row_table[label_name] || Hash.new cell = row_table[label_name] || Hash.new
if cell[:count].nil? if cell[:count].nil?
cell[:count] = 1 cell[:count] = 1
...@@ -38,25 +39,28 @@ class MainController < ApplicationController ...@@ -38,25 +39,28 @@ class MainController < ApplicationController
end end
end end
end end
if row_table[:having_data]
@data_table.append(row_table) @data_table.append(row_table)
end end
end end
end
def update_repo_selected def update_repo_selected
repos_picked = Repository.where(name: params[:selected]) Other.destroy_all(data_type: 0)
repos_picked.each do |repo| params[:selected].each do |selected_repo_name|
repo.selected = true db_selected = Other.new
repo.save db_selected.data = selected_repo_name
end db_selected.data_type = 0
repos_not_picked = Repository.where.not(name: params[:selected]) db_selected.save
repos_not_picked.each do |repo|
repo.selected = false
repo.save
end end
redirect_to action: "index" redirect_to action: "index"
end end
def update_label_selected
Other.destroy_all(data_type: 1)
end
def load_repo_selected def load_repo_selected
# @repos = Repository.find() # @repos = Repository.find()
end end
......
app/helpers/main_helper.rb
...@@ -13,4 +13,25 @@ module MainHelper ...@@ -13,4 +13,25 @@ module MainHelper
mm.to_s + " minutes ago" mm.to_s + " minutes ago"
end end
end end
def get_text_color(bg_color)
r = bg_color[0, 2]
g = bg_color[2, 2]
b = bg_color[4, 2]
r = r.to_i(16)
g = g.to_i(16)
b = b.to_i(16)
a = 1 - (0.299 * r + 0.587 * g + 0.114 * b) / 255;
if a < 0.5
"000000"
else
"ffffff"
end
end
def format_html_id(text)
text.gsub(/[-._]+/, "")
end
end end
app/models/other.rb 0 → 100644
class Other < ApplicationRecord
end
app/views/layouts/application.html.erb
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
<%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %> <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %>
<%= javascript_include_tag 'application', 'data-turbolinks-track': 'reload' %> <%= javascript_include_tag 'application', 'data-turbolinks-track': 'reload' %>
<%= javascript_include_tag 'tableExport' %>
</head> </head>
<body> <body>
......
app/views/main/index.html.erb
...@@ -9,37 +9,9 @@ ...@@ -9,37 +9,9 @@
<span class="icon-bar"></span> <span class="icon-bar"></span>
<span class="icon-bar"></span> <span class="icon-bar"></span>
</button> </button>
<a class="navbar-brand" href="#">GHR</a> <a class="navbar-brand" href="<%= root_url %>">GHR</a>
</div> </div>
<div id="navbar" class="navbar-collapse collapse"> <div id="navbar" class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<li class="active"><a href="#">Home</a></li>
<li><a href="#about">About</a></li>
<li><a href="#contact">Contact</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Dropdown <span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="#">Action</a></li>
<li><a href="#">Another action</a></li>
<li><a href="#">Something else here</a></li>
<li role="separator" class="divider"></li>
<li class="dropdown-header">Nav header</li>
<li><a href="#">Separated link</a></li>
<li><a href="#">One more separated link</a></li>
</ul>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li><a href="../navbar/">Default</a></li>
<li><a href="../navbar-static-top/">Static top</a></li>
<li class="active"><a href="./">Fixed top</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Settings <span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="#">Repositories</a></li>
</ul>
</li>
</ul>
</div> </div>
</div> </div>
</nav> </nav>
...@@ -48,6 +20,7 @@ ...@@ -48,6 +20,7 @@
<div class="container"> <div class="container">
<!-- content --> <!-- content -->
<div class="form-group"> <div class="form-group">
<label>Select repository</label>
<select class="selectpicker" id="selectpicker1" multiple data-live-search="true" multiple data-selected-text-format="count > 3"> <select class="selectpicker" id="selectpicker1" multiple data-live-search="true" multiple data-selected-text-format="count > 3">
<% @repos.each do |repo| %> <% @repos.each do |repo| %>
<option><%= repo.name %></option> <option><%= repo.name %></option>
...@@ -56,6 +29,7 @@ ...@@ -56,6 +29,7 @@
<button id="repo_apply" class="btn btn-default">Apply</button> <button id="repo_apply" class="btn btn-default">Apply</button>
</div> </div>
<div id="toolbar">
<form class="form-inline"> <form class="form-inline">
<div class="form-group"> <div class="form-group">
<label for="from">From</label> <label for="from">From</label>
...@@ -75,9 +49,20 @@ ...@@ -75,9 +49,20 @@
</div> </div>
<input type="submit" class="btn btn-default"> <input type="submit" class="btn btn-default">
</form> </form>
</div>
<div style="margin-top: 20px"> <div style="margin-top: 20px">
<table class="table"> <table data-toolbar="#toolbar"
<thead class="thead-inverse"> data-toggle="table"
data-search="true"
data-show-export="true"
data-show-columns="true"
data-show-export="true"
data-minimum-count-columns="2"
data-show-pagination-switch="true"
data-pagination="true"
data-id-field="id"
data-page-list="[10, 25, 50, 100, ALL]">
<thead>
<tr> <tr>
<th></th> <th></th>
<% @col_label_names.each do |label| %> <% @col_label_names.each do |label| %>
...@@ -91,9 +76,9 @@ ...@@ -91,9 +76,9 @@
<td><%= link_to row[:name], row[:link] %></td> <td><%= link_to row[:name], row[:link] %></td>
<% @col_label_names.each do |label| %> <% @col_label_names.each do |label| %>
<% if row[label].nil? %> <% if row[label].nil? %>
<th>0</th> <td>0</td>
<% else %> <% else %>
<th><a href="" data-toggle="modal" data-target="#<%= row[:name] + label %>"><%= row[label][:count] %></a></th> <td><a href="" data-toggle="modal" data-target="#<%= format_html_id(row[:name] + label) %>"><%= row[label][:count] %></a></td>
<% end %> <% end %>
<% end %> <% end %>
...@@ -103,10 +88,11 @@ ...@@ -103,10 +88,11 @@
</table> </table>
</div> </div>
<% @data_table.each do |row| %> <% @data_table.each do |row| %>
<% @col_label_names.each do |label| %> <% @col_label_names.each do |label| %>
<% if !row[label].nil? %> <% if !row[label].nil? %>
<div id="<%= row[:name] + label %>" class="modal fade bd-example-modal-lg" tabindex="-1" role="dialog" aria-labelledby="myLargeModalLabel" aria-hidden="true"> <div id="<%= format_html_id(row[:name] + label) %>" class="modal fade bd-example-modal-lg" tabindex="-1" role="dialog" aria-labelledby="myLargeModalLabel" aria-hidden="true">
<div class="modal-dialog modal-lg"> <div class="modal-dialog modal-lg">
<div class="modal-content"> <div class="modal-content">
<div class="modal-header"> <div class="modal-header">
...@@ -123,7 +109,7 @@ ...@@ -123,7 +109,7 @@
<td> <td>
<a href="<%= issue[:html_url] %>"><%= issue[:title] %></a> <a href="<%= issue[:html_url] %>"><%= issue[:title] %></a>
<% issue.labels.each do |label| %> <% issue.labels.each do |label| %>
<span class="label label-default" style="background-color: #<%= label[:color] %>"><%= label[:name] %></span> <span class="label label-default" style="background-color: #<%= label[:color] %>; color: #<%= get_text_color(label[:color]) %>;"><%= label[:name] %></span>
<% end %> <% end %>
</td> </td>
<td><%= issue[:created_at] %></td> <td><%= issue[:created_at] %></td>
...@@ -156,6 +142,8 @@ ...@@ -156,6 +142,8 @@
$('#selectpicker1').selectpicker(); // init picker $('#selectpicker1').selectpicker(); // init picker
$('#selectpicker1').selectpicker('val', <%= @selected_repos_name.to_s.html_safe %>); $('#selectpicker1').selectpicker('val', <%= @selected_repos_name.to_s.html_safe %>);
$('#selectpicker2').selectpicker(); // init picker
}); });
$("#repo_apply").click(function() { $("#repo_apply").click(function() {
......
config/initializers/assets.rb
...@@ -9,3 +9,4 @@ Rails.application.config.assets.version = '1.0' ...@@ -9,3 +9,4 @@ Rails.application.config.assets.version = '1.0'
# Precompile additional assets. # Precompile additional assets.
# application.js, application.css, and all non-JS/CSS in app/assets folder are already added. # application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
# Rails.application.config.assets.precompile += %w( search.js ) # Rails.application.config.assets.precompile += %w( search.js )
Rails.application.config.assets.precompile += %w( tableExport.js )
config/schedule.rb 0 → 100644
# Use this file to easily define all of your cron jobs.
#
# It's helpful, but not entirely necessary to understand cron before proceeding.
# http://en.wikipedia.org/wiki/Cron
# Example:
#
# set :output, "/path/to/my/cron_log.log"
#
# every 2.hours do
# command "/usr/bin/some_great_command"
# runner "MyModel.some_method"
# rake "some:great:rake:task"
# end
#
# every 4.days do
# runner "AnotherModel.prune_old_records"
# end
# Learn more: http://github.com/javan/whenever
set :environment, "development"
every 1.hour do
rake "github:insert_github_data"
end
db/migrate/20161129055407_add_html_url_to_issue.rb 0 → 100644
class AddHtmlUrlToIssue < ActiveRecord::Migration[5.0]
def change
add_column :issues, :html_url, :string
end
end
db/migrate/20161130042550_add_open_to_issue.rb 0 → 100644
class AddOpenToIssue < ActiveRecord::Migration[5.0]
def change
add_column :issues, :open, :boolean
end
end
db/migrate/20161202062857_create_others.rb 0 → 100644
class CreateOthers < ActiveRecord::Migration[5.0]
def change
create_table :others do |t|
t.string :data
t.string :data1
t.string :data2
t.integer :data_type
t.timestamps
end
end
end
db/schema.rb
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20161125072404) do ActiveRecord::Schema.define(version: 20161202062857) do
create_table "issues", force: :cascade do |t| create_table "issues", force: :cascade do |t|
t.string "url" t.string "url"
...@@ -22,6 +22,8 @@ ActiveRecord::Schema.define(version: 20161125072404) do ...@@ -22,6 +22,8 @@ ActiveRecord::Schema.define(version: 20161125072404) do
t.integer "user_id" t.integer "user_id"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.string "html_url"
t.boolean "open"
t.index ["repository_id"], name: "index_issues_on_repository_id" t.index ["repository_id"], name: "index_issues_on_repository_id"
t.index ["user_id"], name: "index_issues_on_user_id" t.index ["user_id"], name: "index_issues_on_user_id"
end end
...@@ -51,6 +53,15 @@ ActiveRecord::Schema.define(version: 20161125072404) do ...@@ -51,6 +53,15 @@ ActiveRecord::Schema.define(version: 20161125072404) do
t.index ["repository_id"], name: "index_labels_on_repository_id" t.index ["repository_id"], name: "index_labels_on_repository_id"
end end
create_table "others", force: :cascade do |t|
t.string "data"
t.string "data1"
t.string "data2"
t.integer "data_type"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "repositories", force: :cascade do |t| create_table "repositories", force: :cascade do |t|
t.string "name" t.string "name"
t.string "full_name" t.string "full_name"
......
github.log 0 → 100644
# Logfile created on 2016-12-02 13:00:03 +0700 by logger.rb/54362
I, [2016-12-02T13:00:03.288942 #29541] INFO -- : Begin task insert database from github
I, [2016-12-02T13:34:06.230990 #31710] INFO -- : Begin task insert database from github
I, [2016-12-02T13:34:06.231070 #31710] INFO -- : Insert repos
I, [2016-12-02T13:34:11.550128 #31710] INFO -- : Insert users
I, [2016-12-02T13:34:17.182645 #31710] INFO -- : Insert labels
I, [2016-12-02T13:37:33.545951 #31710] INFO -- : Insert issues
I, [2016-12-02T13:41:30.251754 #31710] INFO -- : begin transaction
I, [2016-12-02T13:41:38.143381 #31710] INFO -- : Finished transaction in 7.891519882s
I, [2016-12-02T13:41:38.143464 #31710] INFO -- : Finished task import in 451.912390186s
I, [2016-12-02T14:00:02.847767 #1467] INFO -- : Begin task insert database from github
I, [2016-12-02T14:00:02.847859 #1467] INFO -- : Insert repos
I, [2016-12-02T14:00:09.153822 #1467] INFO -- : Insert users
I, [2016-12-02T14:00:14.818216 #1467] INFO -- : Insert labels
I, [2016-12-02T14:03:32.620611 #1467] INFO -- : Insert issues
I, [2016-12-02T14:07:32.849635 #1467] INFO -- : begin transaction
I, [2016-12-02T14:07:41.663683 #1467] INFO -- : Finished transaction in 8.81394648s
I, [2016-12-02T14:07:41.663766 #1467] INFO -- : Finished task import in 458.815903477s
I, [2016-12-02T15:00:03.257443 #4602] INFO -- : Begin task insert database from github
I, [2016-12-02T15:00:03.257537 #4602] INFO -- : Insert repos
I, [2016-12-02T15:00:08.729564 #4602] INFO -- : Insert users
I, [2016-12-02T15:00:14.719315 #4602] INFO -- : Insert labels
I, [2016-12-02T15:03:34.365760 #4602] INFO -- : Insert issues
I, [2016-12-02T15:07:33.471646 #4602] INFO -- : begin transaction
I, [2016-12-02T15:07:41.060408 #4602] INFO -- : Finished transaction in 7.588661961s
I, [2016-12-02T15:07:41.060487 #4602] INFO -- : Finished task import in 457.802947713s
I, [2016-12-02T16:00:02.753796 #6484] INFO -- : Begin task insert database from github
I, [2016-12-02T16:00:02.753898 #6484] INFO -- : Insert repos
I, [2016-12-02T16:00:08.666020 #6484] INFO -- : Insert users
I, [2016-12-02T16:00:15.404047 #6484] INFO -- : Insert labels
I, [2016-12-02T16:03:38.395456 #6484] INFO -- : Insert issues
I, [2016-12-02T16:07:35.285332 #6484] INFO -- : begin transaction
I, [2016-12-02T17:00:03.663033 #25408] INFO -- : Begin task insert database from github
I, [2016-12-02T17:00:03.663126 #25408] INFO -- : Insert repos
I, [2016-12-02T17:00:09.540618 #25408] INFO -- : Insert users
I, [2016-12-02T17:00:15.378630 #25408] INFO -- : Insert labels
I, [2016-12-02T17:03:33.861877 #25408] INFO -- : Insert issues
I, [2016-12-02T17:07:38.445501 #25408] INFO -- : begin transaction
I, [2016-12-02T17:07:46.215986 #25408] INFO -- : Finished transaction in 7.770369701s
I, [2016-12-02T17:07:46.216065 #25408] INFO -- : Finished task import in 462.55293774s
lib/tasks/github.rake
namespace :github do namespace :github do
$org = "ZIGExN" $org = "ZIGExN"
$logger = Logger.new("github.log")
desc "Task description"
task task_test: :environment do
# logger = Logger.new("test.log")
# logger.info "a message"
end
desc "Insert Zigexn Github data" desc "Insert Zigexn Github data"
task insert_github_data: :environment do task insert_github_data: :environment do
insert_repos($client) $logger.info "Begin task insert database from github"
insert_users($client) start_time = Time.now
list_db_repo = insert_repos($client)
list_db_user = insert_users($client)
list_db_label = insert_labels($client)
list_db_issue = insert_issues($client, list_db_user, list_db_label)
db_repos = Repository.all $logger.info "begin transaction"
# db_repos = Repository.where(name: "socialtools") start_time_transaction = Time.now
db_repos.each do |db_repo| ActiveRecord::Base.transaction do
insert_labels(db_repo, $client)
Repository.destroy_all
list_db_repo.each do |db_repo|
db_repo.save
end
User.destroy_all
list_db_user.each do |db_user|
db_user.save
end
Label.destroy_all
list_db_label.each do |db_label|
db_label.save
end end
db_repos.each do |db_repo| Issue.destroy_all
insert_issues(db_repo, $client) list_db_issue.each do |db_issue|
db_issue.save
end
end end
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
$logger.info "Finished task import in #{Time.now - start_time}s"
end end
def insert_repos(client) def insert_repos(client)
$logger.info "Insert repos"
repos = client.org_repos($org) repos = client.org_repos($org)
last_page = 1 last_page = 1
unless client.last_response.headers[:link].nil? unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end end
list_db_repo = []
for i in (1..last_page) for i in (1..last_page)
repos = client.org_repos($org, {type: 'all', page: i}) repos = client.org_repos($org, {type: 'all', page: i})
repos.each do |repo| repos.each do |repo|
db_repo = Repository.where(id: repo.id).first
new_flag = false
if db_repo.nil?
db_repo = Repository.new db_repo = Repository.new
new_flag = true
db_repo.id = repo.id db_repo.id = repo.id
db_repo.name = repo.name db_repo.name = repo.name
db_repo.full_name = repo.full_name db_repo.full_name = repo.full_name
...@@ -41,22 +67,18 @@ namespace :github do ...@@ -41,22 +67,18 @@ namespace :github do
db_repo.html_url = repo.html_url db_repo.html_url = repo.html_url
db_repo.created_at = repo.created_at db_repo.created_at = repo.created_at
db_repo.updated_at = repo.updated_at db_repo.updated_at = repo.updated_at
end
if db_repo.save list_db_repo.append(db_repo)
if new_flag
puts "Insert repo success: " + repo.name
else
puts "Update repo success: " + repo.name
end
else
puts "Repo >> An error occured: " + repo.name
end
end end
end end
list_db_repo
end end
def insert_labels(repo, client) def insert_labels(client)
$logger.info "Insert labels"
db_repos = Repository.all
list_db_label = []
db_repos.each do |repo|
labels = client.labels($org + "/" + repo.name) labels = client.labels($org + "/" + repo.name)
last_page = 1 last_page = 1
unless client.last_response.headers[:link].nil? unless client.last_response.headers[:link].nil?
...@@ -66,68 +88,51 @@ namespace :github do ...@@ -66,68 +88,51 @@ namespace :github do
for i in (1..last_page) for i in (1..last_page)
labels = client.labels($org + "/" + repo.name, {page: i}) labels = client.labels($org + "/" + repo.name, {page: i})
labels.each do |label| labels.each do |label|
db_label = Label.where(id: label.id).first
new_flag = false
if db_label.nil?
new_flag = true
db_label = Label.new db_label = Label.new
db_label.id = label.id db_label.id = label.id
db_label.url = label.url db_label.url = label.url
db_label.repository_id = repo.id db_label.repository_id = repo.id
end
db_label.name = label.name db_label.name = label.name
db_label.color = label.color db_label.color = label.color
db_label.default = label.default db_label.default = label.default
if db_label.save list_db_label.append(db_label)
if new_flag
puts "Insert label success: " + label.name
else
puts "Update label success: " + label.name
end
else
puts "Label >> An error occured: " + label.name
end end
end end
end end
list_db_label
end end
def insert_users(client) def insert_users(client)
$logger.info "Insert users"
users = client.org_members($org) users = client.org_members($org)
last_page = 1 last_page = 1
unless client.last_response.headers[:link].nil? unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end end
list_db_user = []
for i in (1..last_page) for i in (1..last_page)
users = client.org_members($org, {page: i}) users = client.org_members($org, {page: i})
users.each do |user| users.each do |user|
db_user = User.where(id: user.id).first
new_flag = false
if db_user.nil?
new_flag = true
db_user = User.new db_user = User.new
db_user.id = user.id db_user.id = user.id
db_user.login = user.login db_user.login = user.login
db_user.url = user.url db_user.url = user.url
db_user.html_url = user.html_url db_user.html_url = user.html_url
end
if db_user.save list_db_user.append(db_user)
if new_flag
puts "Insert user success: " + user.login
else
puts "Update user success: " + user.login
end
else
puts "User >> An error occured: " + user.login
end
end end
end end
list_db_user
end end
def insert_issues(repo, client) def insert_issues(client, list_db_user, list_db_label)
$logger.info "Insert issues"
db_repos = Repository.all
list_db_issue = []
db_repos.each do |repo|
issues = client.list_issues($org + "/" + repo.name) issues = client.list_issues($org + "/" + repo.name)
last_page = 1 last_page = 1
unless client.last_response.headers[:link].nil? unless client.last_response.headers[:link].nil?
...@@ -137,10 +142,6 @@ namespace :github do ...@@ -137,10 +142,6 @@ namespace :github do
for i in (1..last_page) for i in (1..last_page)
issues = client.list_issues($org + "/" + repo.name, {page: i}) issues = client.list_issues($org + "/" + repo.name, {page: i})
issues.each do |issue| issues.each do |issue|
db_issue = Issue.where(id: issue.id).first
new_flag = false
if db_issue.nil?
new_flag = true
db_issue = Issue.new db_issue = Issue.new
db_issue.id = issue.id db_issue.id = issue.id
db_issue.url = issue.url db_issue.url = issue.url
...@@ -148,36 +149,38 @@ namespace :github do ...@@ -148,36 +149,38 @@ namespace :github do
db_issue.number = issue.number db_issue.number = issue.number
db_issue.repository_id = repo.id db_issue.repository_id = repo.id
db_issue.user_id = issue.user.id db_issue.user_id = issue.user.id
end
db_issue.title = issue.title db_issue.title = issue.title
db_issue.body = issue.body db_issue.body = issue.body
db_issue.html_url = issue.html_url
db_issue.created_at = issue.created_at db_issue.created_at = issue.created_at
db_issue.updated_at = issue.updated_at db_issue.updated_at = issue.updated_at
db_issue.labels.clear db_issue.labels.clear
issue.labels.each do |label| issue.labels.each do |label|
db_label = Label.find_by(id: label.id) list_db_label.each do |db_label|
if db_label.id == label.id
db_issue.labels.append(db_label) db_issue.labels.append(db_label)
break
end
end
end end
db_issue.users.clear db_issue.users.clear
issue.assignees.each do |assignee| issue.assignees.each do |assignee|
db_assignee = User.find_by(id: assignee.id) list_db_user.each do |db_user|
db_issue.users.append(db_assignee) if db_user.id == assignee.id
db_issue.users.append(db_user)
break
end
end end
if db_issue.save
if new_flag
puts "Insert issue success: " + issue.title
else
puts "Update issue success: " + issue.title
end end
else
puts "Issue >> an error occured: " + issue.title list_db_issue.append(db_issue)
end end
end end
end end
list_db_issue
end end
end end
test.log 0 → 100644
# Logfile created on 2016-12-02 11:19:00 +0700 by logger.rb/54362
I, [2016-12-02T11:19:07.703083 #25281] INFO -- : hehe
I, [2016-12-02T11:19:52.361449 #25310] INFO -- : hehe
I, [2016-12-02T11:24:03.083031 #25649] INFO -- : a message
I, [2016-12-02T11:25:02.479040 #25730] INFO -- : a message
I, [2016-12-02T11:26:02.943313 #25804] INFO -- : a message
I, [2016-12-02T11:27:03.334729 #25905] INFO -- : a message
I, [2016-12-02T11:28:02.757002 #26087] INFO -- : a message
I, [2016-12-02T11:29:03.248269 #26204] INFO -- : a message
I, [2016-12-02T11:30:02.654350 #26288] INFO -- : a message
I, [2016-12-02T11:31:03.096512 #26515] INFO -- : a message
I, [2016-12-02T11:31:03.111374 #26514] INFO -- : a message
I, [2016-12-02T11:32:02.562251 #26659] INFO -- : a message
I, [2016-12-02T11:32:02.563612 #26658] INFO -- : a message
I, [2016-12-02T11:33:03.022162 #26860] INFO -- : a message
I, [2016-12-02T11:33:03.108266 #26859] INFO -- : a message
I, [2016-12-02T11:34:02.530958 #26973] INFO -- : a message
I, [2016-12-02T12:37:03.189093 #28382] INFO -- : a message
I, [2016-12-02T12:38:02.729618 #28506] INFO -- : a message
I, [2016-12-02T12:40:03.237176 #28785] INFO -- : a message
I, [2016-12-02T12:41:02.772334 #28923] INFO -- : a message
test/fixtures/others.yml 0 → 100644
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
one:
data: MyString
data1: MyString
data2: MyString
data_type: 1
two:
data: MyString
data1: MyString
data2: MyString
data_type: 1
test/models/other_test.rb 0 → 100644
require 'test_helper'
class OtherTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment