Skip to content

V
ventura-ghr
Commit fbdb9b04 by Bui Minh Duc
Options

optimize github import

parent 29b46411
Showing with 282 additions and 117 deletions
Gemfile
......@@ -37,6 +37,7 @@ gem 'bootstrap-select-rails', '~> 1.6', '>= 1.6.3'
gem 'bootstrap-table-rails', '~> 1.11'
gem 'momentjs-rails', '>= 2.9.0'
gem 'bootstrap3-datetimepicker-rails', '~> 4.14.30'
gem 'whenever', '~> 0.9.7'
group :development, :test do
# Call 'byebug' anywhere in the code to stop execution and get a debugger console
......
Gemfile.lock
......@@ -52,6 +52,7 @@ GEM
momentjs-rails (>= 2.8.1)
builder (3.2.2)
byebug (9.0.6)
chronic (0.10.2)
coffee-rails (4.2.1)
coffee-script (>= 2.2.0)
railties (>= 4.0.0, < 5.2.x)
......@@ -171,6 +172,8 @@ GEM
websocket-driver (0.6.4)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.2)
whenever (0.9.7)
chronic (>= 0.6.3)
will_paginate (3.1.5)
PLATFORMS
......@@ -198,6 +201,7 @@ DEPENDENCIES
tzinfo-data
uglifier (>= 1.3.0)
web-console
whenever (~> 0.9.7)
will_paginate (~> 3.1, >= 3.1.5)
BUNDLED WITH
......
app/assets/javascripts/application.js
......@@ -19,5 +19,6 @@
//= require bootstrap/alert
//= require bootstrap/dropdown
//= require bootstrap-table
//= require extensions/bootstrap-table-export.js
//= require moment
//= require bootstrap-datetimepicker
app/controllers/main_controller.rb
......@@ -2,26 +2,27 @@ class MainController < ApplicationController
def index
@repos = @repos || Repository.all
@selected_repos = Repository.where(selected: true)
@selected_repos_name = Other.where(data_type: 0).map{ |other| other.data }
@selected_repos = Repository.where(name: @selected_repos_name)
@label_filter = ["ventura", "priority-ugent", "priority-high", "priority-low"]
@col_label_names = [
"discussion", "todo", "inprogress",
"vnreview", "jpreview", "ready",
"done", "releasefailed", "pending", "ventura"]
"done", "releasefailed", "pending", "ventura", "prioritylow", "priorityhigh", "priorityurgent"]
@selected_repos_name = []
@data_table = []
@selected_repos.each do |repo|
@selected_repos_name.append(repo.name)
row_table = Hash.new
row_table[:name] = repo.name
row_table[:link] = repo.html_url
row_table[:having_data] = false
repo.issues.each do |issue|
issue.labels.each do |label|
label_name = standardize_string(label[:name])
@col_label_names.each do |col_label|
if label_name == col_label
row_table[:having_data] = true
cell = row_table[label_name] || Hash.new
if cell[:count].nil?
cell[:count] = 1
......@@ -38,25 +39,28 @@ class MainController < ApplicationController
end
end
end
if row_table[:having_data]
@data_table.append(row_table)
end
end
end
def update_repo_selected
repos_picked = Repository.where(name: params[:selected])
repos_picked.each do |repo|
repo.selected = true
repo.save
end
repos_not_picked = Repository.where.not(name: params[:selected])
repos_not_picked.each do |repo|
repo.selected = false
repo.save
Other.destroy_all(data_type: 0)
params[:selected].each do |selected_repo_name|
db_selected = Other.new
db_selected.data = selected_repo_name
db_selected.data_type = 0
db_selected.save
end
redirect_to action: "index"
end
def update_label_selected
Other.destroy_all(data_type: 1)
end
def load_repo_selected
# @repos = Repository.find()
end
......
app/helpers/main_helper.rb
......@@ -13,4 +13,25 @@ module MainHelper
mm.to_s + " minutes ago"
end
end
def get_text_color(bg_color)
r = bg_color[0, 2]
g = bg_color[2, 2]
b = bg_color[4, 2]
r = r.to_i(16)
g = g.to_i(16)
b = b.to_i(16)
a = 1 - (0.299 * r + 0.587 * g + 0.114 * b) / 255;
if a < 0.5
"000000"
else
"ffffff"
end
end
def format_html_id(text)
text.gsub(/[-._]+/, "")
end
end
app/models/other.rb 0 → 100644
class Other < ApplicationRecord
end
app/views/layouts/application.html.erb
......@@ -6,6 +6,7 @@
<%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %>
<%= javascript_include_tag 'application', 'data-turbolinks-track': 'reload' %>
<%= javascript_include_tag 'tableExport' %>
</head>
<body>
......
app/views/main/index.html.erb
......@@ -9,37 +9,9 @@
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="#">GHR</a>
<a class="navbar-brand" href="<%= root_url %>">GHR</a>
</div>
<div id="navbar" class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<li class="active"><a href="#">Home</a></li>
<li><a href="#about">About</a></li>
<li><a href="#contact">Contact</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Dropdown <span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="#">Action</a></li>
<li><a href="#">Another action</a></li>
<li><a href="#">Something else here</a></li>
<li role="separator" class="divider"></li>
<li class="dropdown-header">Nav header</li>
<li><a href="#">Separated link</a></li>
<li><a href="#">One more separated link</a></li>
</ul>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li><a href="../navbar/">Default</a></li>
<li><a href="../navbar-static-top/">Static top</a></li>
<li class="active"><a href="./">Fixed top</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Settings <span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="#">Repositories</a></li>
</ul>
</li>
</ul>
</div>
</div>
</nav>
......@@ -48,6 +20,7 @@
<div class="container">
<!-- content -->
<div class="form-group">
<label>Select repository</label>
<select class="selectpicker" id="selectpicker1" multiple data-live-search="true" multiple data-selected-text-format="count > 3">
<% @repos.each do |repo| %>
<option><%= repo.name %></option>
......@@ -56,6 +29,7 @@
<button id="repo_apply" class="btn btn-default">Apply</button>
</div>
<div id="toolbar">
<form class="form-inline">
<div class="form-group">
<label for="from">From</label>
......@@ -75,9 +49,20 @@
</div>
<input type="submit" class="btn btn-default">
</form>
</div>
<div style="margin-top: 20px">
<table class="table">
<thead class="thead-inverse">
<table data-toolbar="#toolbar"
data-toggle="table"
data-search="true"
data-show-export="true"
data-show-columns="true"
data-show-export="true"
data-minimum-count-columns="2"
data-show-pagination-switch="true"
data-pagination="true"
data-id-field="id"
data-page-list="[10, 25, 50, 100, ALL]">
<thead>
<tr>
<th></th>
<% @col_label_names.each do |label| %>
......@@ -91,9 +76,9 @@
<td><%= link_to row[:name], row[:link] %></td>
<% @col_label_names.each do |label| %>
<% if row[label].nil? %>
<th>0</th>
<td>0</td>
<% else %>
<th><a href="" data-toggle="modal" data-target="#<%= row[:name] + label %>"><%= row[label][:count] %></a></th>
<td><a href="" data-toggle="modal" data-target="#<%= format_html_id(row[:name] + label) %>"><%= row[label][:count] %></a></td>
<% end %>
<% end %>
......@@ -103,10 +88,11 @@
</table>
</div>
<% @data_table.each do |row| %>
<% @col_label_names.each do |label| %>
<% if !row[label].nil? %>
<div id="<%= row[:name] + label %>" class="modal fade bd-example-modal-lg" tabindex="-1" role="dialog" aria-labelledby="myLargeModalLabel" aria-hidden="true">
<div id="<%= format_html_id(row[:name] + label) %>" class="modal fade bd-example-modal-lg" tabindex="-1" role="dialog" aria-labelledby="myLargeModalLabel" aria-hidden="true">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
......@@ -123,7 +109,7 @@
<td>
<a href="<%= issue[:html_url] %>"><%= issue[:title] %></a>
<% issue.labels.each do |label| %>
<span class="label label-default" style="background-color: #<%= label[:color] %>"><%= label[:name] %></span>
<span class="label label-default" style="background-color: #<%= label[:color] %>; color: #<%= get_text_color(label[:color]) %>;"><%= label[:name] %></span>
<% end %>
</td>
<td><%= issue[:created_at] %></td>
......@@ -156,6 +142,8 @@
$('#selectpicker1').selectpicker(); // init picker
$('#selectpicker1').selectpicker('val', <%= @selected_repos_name.to_s.html_safe %>);
$('#selectpicker2').selectpicker(); // init picker
});
$("#repo_apply").click(function() {
......
config/initializers/assets.rb
......@@ -9,3 +9,4 @@ Rails.application.config.assets.version = '1.0'
# Precompile additional assets.
# application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
# Rails.application.config.assets.precompile += %w( search.js )
Rails.application.config.assets.precompile += %w( tableExport.js )
config/schedule.rb 0 → 100644
# Use this file to easily define all of your cron jobs.
#
# It's helpful, but not entirely necessary to understand cron before proceeding.
# http://en.wikipedia.org/wiki/Cron
# Example:
#
# set :output, "/path/to/my/cron_log.log"
#
# every 2.hours do
# command "/usr/bin/some_great_command"
# runner "MyModel.some_method"
# rake "some:great:rake:task"
# end
#
# every 4.days do
# runner "AnotherModel.prune_old_records"
# end
# Learn more: http://github.com/javan/whenever
set :environment, "development"
every 1.hour do
rake "github:insert_github_data"
end
db/migrate/20161129055407_add_html_url_to_issue.rb 0 → 100644
class AddHtmlUrlToIssue < ActiveRecord::Migration[5.0]
def change
add_column :issues, :html_url, :string
end
end
db/migrate/20161130042550_add_open_to_issue.rb 0 → 100644
class AddOpenToIssue < ActiveRecord::Migration[5.0]
def change
add_column :issues, :open, :boolean
end
end
db/migrate/20161202062857_create_others.rb 0 → 100644
class CreateOthers < ActiveRecord::Migration[5.0]
def change
create_table :others do |t|
t.string :data
t.string :data1
t.string :data2
t.integer :data_type
t.timestamps
end
end
end
db/schema.rb
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20161125072404) do
ActiveRecord::Schema.define(version: 20161202062857) do
create_table "issues", force: :cascade do |t|
t.string "url"
......@@ -22,6 +22,8 @@ ActiveRecord::Schema.define(version: 20161125072404) do
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "html_url"
t.boolean "open"
t.index ["repository_id"], name: "index_issues_on_repository_id"
t.index ["user_id"], name: "index_issues_on_user_id"
end
......@@ -51,6 +53,15 @@ ActiveRecord::Schema.define(version: 20161125072404) do
t.index ["repository_id"], name: "index_labels_on_repository_id"
end
create_table "others", force: :cascade do |t|
t.string "data"
t.string "data1"
t.string "data2"
t.integer "data_type"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
create_table "repositories", force: :cascade do |t|
t.string "name"
t.string "full_name"
......
github.log 0 → 100644
# Logfile created on 2016-12-02 13:00:03 +0700 by logger.rb/54362
I, [2016-12-02T13:00:03.288942 #29541] INFO -- : Begin task insert database from github
I, [2016-12-02T13:34:06.230990 #31710] INFO -- : Begin task insert database from github
I, [2016-12-02T13:34:06.231070 #31710] INFO -- : Insert repos
I, [2016-12-02T13:34:11.550128 #31710] INFO -- : Insert users
I, [2016-12-02T13:34:17.182645 #31710] INFO -- : Insert labels
I, [2016-12-02T13:37:33.545951 #31710] INFO -- : Insert issues
I, [2016-12-02T13:41:30.251754 #31710] INFO -- : begin transaction
I, [2016-12-02T13:41:38.143381 #31710] INFO -- : Finished transaction in 7.891519882s
I, [2016-12-02T13:41:38.143464 #31710] INFO -- : Finished task import in 451.912390186s
I, [2016-12-02T14:00:02.847767 #1467] INFO -- : Begin task insert database from github
I, [2016-12-02T14:00:02.847859 #1467] INFO -- : Insert repos
I, [2016-12-02T14:00:09.153822 #1467] INFO -- : Insert users
I, [2016-12-02T14:00:14.818216 #1467] INFO -- : Insert labels
I, [2016-12-02T14:03:32.620611 #1467] INFO -- : Insert issues
I, [2016-12-02T14:07:32.849635 #1467] INFO -- : begin transaction
I, [2016-12-02T14:07:41.663683 #1467] INFO -- : Finished transaction in 8.81394648s
I, [2016-12-02T14:07:41.663766 #1467] INFO -- : Finished task import in 458.815903477s
I, [2016-12-02T15:00:03.257443 #4602] INFO -- : Begin task insert database from github
I, [2016-12-02T15:00:03.257537 #4602] INFO -- : Insert repos
I, [2016-12-02T15:00:08.729564 #4602] INFO -- : Insert users
I, [2016-12-02T15:00:14.719315 #4602] INFO -- : Insert labels
I, [2016-12-02T15:03:34.365760 #4602] INFO -- : Insert issues
I, [2016-12-02T15:07:33.471646 #4602] INFO -- : begin transaction
I, [2016-12-02T15:07:41.060408 #4602] INFO -- : Finished transaction in 7.588661961s
I, [2016-12-02T15:07:41.060487 #4602] INFO -- : Finished task import in 457.802947713s
I, [2016-12-02T16:00:02.753796 #6484] INFO -- : Begin task insert database from github
I, [2016-12-02T16:00:02.753898 #6484] INFO -- : Insert repos
I, [2016-12-02T16:00:08.666020 #6484] INFO -- : Insert users
I, [2016-12-02T16:00:15.404047 #6484] INFO -- : Insert labels
I, [2016-12-02T16:03:38.395456 #6484] INFO -- : Insert issues
I, [2016-12-02T16:07:35.285332 #6484] INFO -- : begin transaction
I, [2016-12-02T17:00:03.663033 #25408] INFO -- : Begin task insert database from github
I, [2016-12-02T17:00:03.663126 #25408] INFO -- : Insert repos
I, [2016-12-02T17:00:09.540618 #25408] INFO -- : Insert users
I, [2016-12-02T17:00:15.378630 #25408] INFO -- : Insert labels
I, [2016-12-02T17:03:33.861877 #25408] INFO -- : Insert issues
I, [2016-12-02T17:07:38.445501 #25408] INFO -- : begin transaction
I, [2016-12-02T17:07:46.215986 #25408] INFO -- : Finished transaction in 7.770369701s
I, [2016-12-02T17:07:46.216065 #25408] INFO -- : Finished task import in 462.55293774s
lib/tasks/github.rake
namespace :github do
$org = "ZIGExN"
$logger = Logger.new("github.log")
desc "Task description"
task task_test: :environment do
# logger = Logger.new("test.log")
# logger.info "a message"
end
desc "Insert Zigexn Github data"
task insert_github_data: :environment do
insert_repos($client)
insert_users($client)
$logger.info "Begin task insert database from github"
start_time = Time.now
list_db_repo = insert_repos($client)
list_db_user = insert_users($client)
list_db_label = insert_labels($client)
list_db_issue = insert_issues($client, list_db_user, list_db_label)
db_repos = Repository.all
# db_repos = Repository.where(name: "socialtools")
db_repos.each do |db_repo|
insert_labels(db_repo, $client)
$logger.info "begin transaction"
start_time_transaction = Time.now
ActiveRecord::Base.transaction do
Repository.destroy_all
list_db_repo.each do |db_repo|
db_repo.save
end
User.destroy_all
list_db_user.each do |db_user|
db_user.save
end
Label.destroy_all
list_db_label.each do |db_label|
db_label.save
end
db_repos.each do |db_repo|
insert_issues(db_repo, $client)
Issue.destroy_all
list_db_issue.each do |db_issue|
db_issue.save
end
end
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
$logger.info "Finished task import in #{Time.now - start_time}s"
end
def insert_repos(client)
$logger.info "Insert repos"
repos = client.org_repos($org)
last_page = 1
unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
list_db_repo = []
for i in (1..last_page)
repos = client.org_repos($org, {type: 'all', page: i})
repos.each do |repo|
db_repo = Repository.where(id: repo.id).first
new_flag = false
if db_repo.nil?
db_repo = Repository.new
new_flag = true
db_repo.id = repo.id
db_repo.name = repo.name
db_repo.full_name = repo.full_name
......@@ -41,22 +67,18 @@ namespace :github do
db_repo.html_url = repo.html_url
db_repo.created_at = repo.created_at
db_repo.updated_at = repo.updated_at
end
if db_repo.save
if new_flag
puts "Insert repo success: " + repo.name
else
puts "Update repo success: " + repo.name
end
else
puts "Repo >> An error occured: " + repo.name
end
list_db_repo.append(db_repo)
end
end
list_db_repo
end
def insert_labels(repo, client)
def insert_labels(client)
$logger.info "Insert labels"
db_repos = Repository.all
list_db_label = []
db_repos.each do |repo|
labels = client.labels($org + "/" + repo.name)
last_page = 1
unless client.last_response.headers[:link].nil?
......@@ -66,68 +88,51 @@ namespace :github do
for i in (1..last_page)
labels = client.labels($org + "/" + repo.name, {page: i})
labels.each do |label|
db_label = Label.where(id: label.id).first
new_flag = false
if db_label.nil?
new_flag = true
db_label = Label.new
db_label.id = label.id
db_label.url = label.url
db_label.repository_id = repo.id
end
db_label.name = label.name
db_label.color = label.color
db_label.default = label.default
if db_label.save
if new_flag
puts "Insert label success: " + label.name
else
puts "Update label success: " + label.name
end
else
puts "Label >> An error occured: " + label.name
list_db_label.append(db_label)
end
end
end
list_db_label
end
def insert_users(client)
$logger.info "Insert users"
users = client.org_members($org)
last_page = 1
unless client.last_response.headers[:link].nil?
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
list_db_user = []
for i in (1..last_page)
users = client.org_members($org, {page: i})
users.each do |user|
db_user = User.where(id: user.id).first
new_flag = false
if db_user.nil?
new_flag = true
db_user = User.new
db_user.id = user.id
db_user.login = user.login
db_user.url = user.url
db_user.html_url = user.html_url
end
if db_user.save
if new_flag
puts "Insert user success: " + user.login
else
puts "Update user success: " + user.login
end
else
puts "User >> An error occured: " + user.login
end
list_db_user.append(db_user)
end
end
list_db_user
end
def insert_issues(repo, client)
def insert_issues(client, list_db_user, list_db_label)
$logger.info "Insert issues"
db_repos = Repository.all
list_db_issue = []
db_repos.each do |repo|
issues = client.list_issues($org + "/" + repo.name)
last_page = 1
unless client.last_response.headers[:link].nil?
......@@ -137,10 +142,6 @@ namespace :github do
for i in (1..last_page)
issues = client.list_issues($org + "/" + repo.name, {page: i})
issues.each do |issue|
db_issue = Issue.where(id: issue.id).first
new_flag = false
if db_issue.nil?
new_flag = true
db_issue = Issue.new
db_issue.id = issue.id
db_issue.url = issue.url
......@@ -148,36 +149,38 @@ namespace :github do
db_issue.number = issue.number
db_issue.repository_id = repo.id
db_issue.user_id = issue.user.id
end
db_issue.title = issue.title
db_issue.body = issue.body
db_issue.html_url = issue.html_url
db_issue.created_at = issue.created_at
db_issue.updated_at = issue.updated_at
db_issue.labels.clear
issue.labels.each do |label|
db_label = Label.find_by(id: label.id)
list_db_label.each do |db_label|
if db_label.id == label.id
db_issue.labels.append(db_label)
break
end
end
end
db_issue.users.clear
issue.assignees.each do |assignee|
db_assignee = User.find_by(id: assignee.id)
db_issue.users.append(db_assignee)
list_db_user.each do |db_user|
if db_user.id == assignee.id
db_issue.users.append(db_user)
break
end
end
if db_issue.save
if new_flag
puts "Insert issue success: " + issue.title
else
puts "Update issue success: " + issue.title
end
else
puts "Issue >> an error occured: " + issue.title
list_db_issue.append(db_issue)
end
end
end
list_db_issue
end
end
test.log 0 → 100644
# Logfile created on 2016-12-02 11:19:00 +0700 by logger.rb/54362
I, [2016-12-02T11:19:07.703083 #25281] INFO -- : hehe
I, [2016-12-02T11:19:52.361449 #25310] INFO -- : hehe
I, [2016-12-02T11:24:03.083031 #25649] INFO -- : a message
I, [2016-12-02T11:25:02.479040 #25730] INFO -- : a message
I, [2016-12-02T11:26:02.943313 #25804] INFO -- : a message
I, [2016-12-02T11:27:03.334729 #25905] INFO -- : a message
I, [2016-12-02T11:28:02.757002 #26087] INFO -- : a message
I, [2016-12-02T11:29:03.248269 #26204] INFO -- : a message
I, [2016-12-02T11:30:02.654350 #26288] INFO -- : a message
I, [2016-12-02T11:31:03.096512 #26515] INFO -- : a message
I, [2016-12-02T11:31:03.111374 #26514] INFO -- : a message
I, [2016-12-02T11:32:02.562251 #26659] INFO -- : a message
I, [2016-12-02T11:32:02.563612 #26658] INFO -- : a message
I, [2016-12-02T11:33:03.022162 #26860] INFO -- : a message
I, [2016-12-02T11:33:03.108266 #26859] INFO -- : a message
I, [2016-12-02T11:34:02.530958 #26973] INFO -- : a message
I, [2016-12-02T12:37:03.189093 #28382] INFO -- : a message
I, [2016-12-02T12:38:02.729618 #28506] INFO -- : a message
I, [2016-12-02T12:40:03.237176 #28785] INFO -- : a message
I, [2016-12-02T12:41:02.772334 #28923] INFO -- : a message
test/fixtures/others.yml 0 → 100644
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
one:
data: MyString
data1: MyString
data2: MyString
data_type: 1
two:
data: MyString
data1: MyString
data2: MyString
data_type: 1
test/models/other_test.rb 0 → 100644
require 'test_helper'
class OtherTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment