Skip to content

V
ventura-ghr
Commit 4f7c94b1 by Bui Minh Duc
Options

migrate to mysql

parent 4e21255b
Showing with 530 additions and 223 deletions
Gemfile
......@@ -29,6 +29,7 @@ gem 'jbuilder', '~> 2.5'
# Use Capistrano for deployment
# gem 'capistrano-rails', group: :development
gem 'mysql2', '~> 0.4.5'
gem 'bootstrap-sass', '~> 3.3.6'
gem 'octokit', '~> 4.6', '>= 4.6.2'
......
Gemfile.lock
......@@ -96,6 +96,7 @@ GEM
railties (>= 3.1)
multi_json (1.12.1)
multipart-post (2.0.0)
mysql2 (0.4.5)
nio4r (1.2.1)
nokogiri (1.7.0)
mini_portile2 (~> 2.1.0)
......@@ -199,6 +200,7 @@ DEPENDENCIES
jquery-rails
listen (~> 3.0.5)
momentjs-rails (>= 2.9.0)
mysql2 (~> 0.4.5)
octokit (~> 4.6, >= 4.6.2)
puma (~> 3.0)
rails (~> 5.0.0, >= 5.0.0.1)
......
app/controllers/main_controller.rb
......@@ -32,7 +32,7 @@ class MainController < ApplicationController
@selected_repos.each do |repo|
row_table = Hash.new
row_table[:name] = repo.name
row_table[:link] = repo.html_url
row_table[:repo_id] = repo.id
row_table[:having_data] = false
repo.issues.each do |issue|
if session[:filter_date].nil? || session[:filter_date] == "" || Date.parse(issue.created_at.to_s) >= Date.parse(session[:filter_date])
......
app/controllers/repositories_controller.rb
......@@ -4,6 +4,26 @@ class RepositoriesController < ApplicationController
@repos = Repository.all
end
def show
# byebug
@type = {
0 => "Open",
1 => "Closed",
2 => "Both"
}
@current_type = params["issue"]["type"]
@repo = Repository.find_by(id: params[:id])
@issues = nil
if @current_type.nil? || @current_type == "0"
@issues = @repo.issues.where("issues.closed_at IS NULL AND issues.is_pull = ?", false)
elsif @current_type == "1"
@issues = @repo.issues.where("issues.closed_at IS NOT NULL AND issues.is_pull = ?", false)
else
@issues = @repo.issues.where(is_pull: false)
end
end
def update
@repo = Repository.find_by(id: params[:id])
RepositoryJob.perform_now(@repo.name)
......
app/controllers/settings_controller.rb
class SettingsController < ApplicationController
def index
@repos = Repository.all
@repos_to_update = Other.where(data_type: 2).map{ |x| x.data }
end
def update_setting_1
$repos_to_update = params[:selected]
@repos_to_update = params[:selected]
if !@repos_to_update.nil? && @repos_to_update.class == Array
Other.where(data_type: 2).delete_all
@repos_to_update.each do |repo|
Other.create(data: repo, data_type: 2)
end
end
end
end
app/helpers/application_helper.rb
module ApplicationHelper
def format_duration_time(int_time)
mm, ss = int_time.divmod(60)
hh, mm = mm.divmod(60)
dd, hh = hh.divmod(24)
if dd != 0
dd.to_s + " days " + hh.to_s + " hours"
elsif hh != 0
hh.to_s + " hours " + mm.to_s + " minutes"
else
mm.to_s + " minutes " + ss.to_s + " seconds"
end
end
end
app/helpers/issue_helper.rb
module IssueHelper
include ApplicationHelper
end
app/views/issues/show.html.erb
......@@ -42,98 +42,96 @@
</div>
<% end %>
<% end %>
<table class="table table-bordered">
<thead>
<tr>
<th class="col-md-2">Label</th>
<th class="col-md-3">Duration</th>
<th class="col-md-7">Time</th>
</tr>
</thead>
<tbody>
<% @label_data.each do |key, value| %>
<tr>
<td><%= key %></td>
<% if value.nil? %>
<td>---</td>
<td>---</td>
<% else %>
<td><%= format_duration_time(value["time"].to_i) %></td>
<td>
<select>
<% value["data"].each do |x| %>
<option><%= x.started_time.to_s + " - " + x.end_time.to_s %></option>
<% end %>
</select>
</td>
<% end %>
</tr>
<% end %>
</tbody>
</table>
</div>
<div class="col-md-4">
<div>
<div class="panel panel-default">
<div class="panel-heading">
<h4>Assignee</h4>
</div>
<ul class="list-group">
<% @issue.users.each do |user| %>
<li class="list-group-item"><%= user.login %></li>
<% end %>
</ul>
<div class="panel panel-default">
<div class="panel-heading">
<h4>Assignee</h4>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4>Label</h4>
</div>
<ul class="list-group">
<% @issue.labels.each do |label| %>
<li class="list-group-item"><%= label.name %></li>
<% end %>
</ul>
<ul class="list-group">
<% @issue.users.each do |user| %>
<li class="list-group-item"><%= user.login %></li>
<% end %>
</ul>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4>Label</h4>
</div>
<table class="table table-bordered">
<tbody>
<tr>
<td class="col-md-4">Started Time</td>
<td class="col-md-8">
<%= @started_time %>
</td>
</tr>
<tr>
<td class="col-md-4">End Time</td>
<td class="col-md-8">
<%= @end_time %>
</td>
</tr>
<tr>
<td class="col-md-4">Duration</td>
<td class="col-md-8">
<%= @duration %>
</td>
</tr>
<tr>
<td class="col-md-4">Estimate</td>
<td class="col-md-8">15 days</td>
</tr>
</tbody>
</table>
<table class="table table-bordered">
<thead>
<tr>
<th class="col-md-4">Label</th>
<th class="col-md-4">Duration</th>
<th class="col-md-4">Time</th>
</tr>
</thead>
<tbody>
<% @label_data.each do |key, value| %>
<tr>
<td><%= key %></td>
<% if value.nil? %>
<td>---</td>
<td>---</td>
<% else %>
<td><%= value["time"].to_i %></td>
<td>
<select>
<% value["data"].each do |x| %>
<option><%= x.started_time.to_s + " - " + x.end_time.to_s %></option>
<% end %>
</select>
</td>
<% end %>
</tr>
<% end %>
</tbody>
</table>
<div>
<div class="panel panel-default">
<div class="panel-heading">
<h4>Pull Requests</h4>
</div>
<ul class="list-group">
<% unless @pulls.nil? %>
<% @pulls.each do |pull| %>
<li class="list-group-item"><%= link_to pull.title, issue_path(pull) %></li>
<% end %>
<% end %>
</ul>
<ul class="list-group">
<% @issue.labels.each do |label| %>
<li class="list-group-item"><%= label.name %></li>
<% end %>
</ul>
</div>
<table class="table table-bordered">
<tbody>
<tr>
<td class="col-md-4">Started Time</td>
<td class="col-md-8">
<%= @started_time %>
</td>
</tr>
<tr>
<td class="col-md-4">End Time</td>
<td class="col-md-8">
<%= @end_time %>
</td>
</tr>
<tr>
<td class="col-md-4">Duration</td>
<td class="col-md-8">
<%= format_duration_time(@duration.to_i) %>
</td>
</tr>
<tr>
<td class="col-md-4">Estimate</td>
<td class="col-md-8">15 days</td>
</tr>
</tbody>
</table>
<div class="panel panel-default">
<div class="panel-heading">
<h4>Pull Requests</h4>
</div>
<ul class="list-group">
<% unless @pulls.nil? %>
<% @pulls.each do |pull| %>
<li class="list-group-item"><%= link_to pull.title, issue_path(pull) %></li>
<% end %>
<% end %>
</ul>
</div>
</div>
</div>
......
app/views/main/index.html.erb
......@@ -62,7 +62,7 @@
<tbody>
<% @data_table.each do |row| %>
<tr>
<td class="col-md-2"><%= link_to row[:name], row[:link] %></td>
<td class="col-md-2"><%= link_to row[:name], repository_path(row[:repo_id]) %></td>
<% @col_label_names.each do |label| %>
<% if row[label].nil? %>
<td class="col-md-1">0</td>
......
app/views/repositories/index.html.erb
......@@ -28,7 +28,7 @@
<tbody>
<% @repos.each do |repo| %>
<tr>
<td><%= repo.name %></td>
<td><%= link_to repo.name, repository_path(repo.id) %></td>
<td><%= repo.description %></td>
<td>
<%= form_for :repo, url: { action: "update", id: repo.id }, method: :patch, html: { class: "form-inline" } do |f| %>
......
app/views/repositories/show.html.erb 0 → 100644
<div class="container">
<h2><%= @repo.name %></h2>
<div id="toolbar">
<form class="form-inline">
<div class="form-group">
<label>Type</label>
<div class="form-group">
<%= select("issue", "type", @type.collect{ |x| [x[1], x[0]] }, {}, { class: "form-control" }) %>
<input type="submit" class="btn btn-default">
</div>
</div>
</form>
</div>
<div style="margin-top: 20px">
<table data-toolbar="#toolbar"
data-toggle="table"
data-search="true"
data-show-export="true"
data-show-columns="true"
data-show-export="true"
data-minimum-count-columns="2"
data-show-pagination-switch="true"
data-pagination="true"
data-id-field="id"
data-page-list="[10, 25, 50, 100, ALL]">
<thead>
<tr>
<th>List of issues</th>
</tr>
</thead>
<tbody>
<% @issues.each do |issue| %>
<tr>
<td><%= link_to issue.title, issue %></td>
</tr>
<% end %>
</tbody>
</table>
</div>
</div>
app/views/settings/index.html.erb
......@@ -12,8 +12,8 @@
</select>
<button class="btn btn-default" id="btn1">Save</button>
<br><br>
<% if $repos_to_update.class == Array %>
<% $repos_to_update.each do |repo| %>
<% if @repos_to_update.class == Array %>
<% @repos_to_update.each do |repo| %>
<span class="label label-primary"><%= repo %></span>
<% end %>
<% end %>
......@@ -24,8 +24,8 @@
<script type="text/javascript">
$(document).ready(function() {
$('#selectpicker').selectpicker(); // init picker
<% if $repos_to_update.class == Array %>
$('#selectpicker').selectpicker('val', <%= $repos_to_update.to_s.html_safe %>);
<% if @repos_to_update.class == Array %>
$('#selectpicker').selectpicker('val', <%= @repos_to_update.to_s.html_safe %>);
<% end %>
});
......
config/database.yml
......@@ -5,21 +5,25 @@
# gem 'sqlite3'
#
default: &default
adapter: sqlite3
adapter: mysql2
encoding: utf8mb4
collation: utf8mb4_bin
database: ghr
username: root
password: 12345678
host: 127.0.0.1
port: 3306
pool: 5
timeout: 5000
development:
<<: *default
database: db/development.sqlite3
# Warning: The database defined as "test" will be erased and
# re-generated from your development database when you run "rake".
# Do not set this db to the same as development or production.
test:
<<: *default
database: db/test.sqlite3
production:
<<: *default
database: db/production.sqlite3
config/initializers/github.rb
$client = Octokit::Client.new(access_token: Rails.application.secrets.GITHUB_TOKEN)
$repos_to_update = []
# $repos_to_update = []
db/migrate/20161125072356_create_issues.rb
......@@ -7,7 +7,7 @@ class CreateIssues < ActiveRecord::Migration[5.0]
t.string :title
t.text :body
t.references :repository, foreign_key: true
t.references :user, foreign_key: true
t.references :user, foreign_key: true, null: true
t.date :created_at
t.date :updated_at
......
db/migrate/20170105071411_change_collumn_body_in_review_comment.rb 0 → 100644
class ChangeCollumnBodyInReviewComment < ActiveRecord::Migration[5.0]
def change
change_column :review_comments, :body, :text
end
end
db/migrate/20170105072651_change_collumn_diff_hub_in_review_comment.rb 0 → 100644
class ChangeCollumnDiffHubInReviewComment < ActiveRecord::Migration[5.0]
def change
change_column :review_comments, :diff_hunk, :text
end
end
db/schema.rb
......@@ -10,55 +10,55 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20161230092006) do
ActiveRecord::Schema.define(version: 20170105072651) do
create_table "comments", force: :cascade do |t|
create_table "comments", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "html_url"
t.text "body"
t.text "body", limit: 65535
t.integer "issue_id"
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.integer "score"
t.index ["issue_id"], name: "index_comments_on_issue_id"
t.index ["user_id"], name: "index_comments_on_user_id"
t.index ["issue_id"], name: "index_comments_on_issue_id", using: :btree
t.index ["user_id"], name: "index_comments_on_user_id", using: :btree
end
create_table "issues", force: :cascade do |t|
create_table "issues", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "url"
t.string "repository_url"
t.integer "number"
t.string "title"
t.text "body"
t.text "body", limit: 65535
t.integer "repository_id"
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "html_url"
t.boolean "open"
t.date "closed_at"
t.boolean "is_pull"
t.integer "issue_id"
t.index ["issue_id"], name: "index_issues_on_issue_id"
t.index ["repository_id"], name: "index_issues_on_repository_id"
t.index ["user_id"], name: "index_issues_on_user_id"
t.index ["issue_id"], name: "index_issues_on_issue_id", using: :btree
t.index ["repository_id"], name: "index_issues_on_repository_id", using: :btree
t.index ["user_id"], name: "index_issues_on_user_id", using: :btree
end
create_table "issues_labels", id: false, force: :cascade do |t|
create_table "issues_labels", id: false, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.integer "issue_id", null: false
t.integer "label_id", null: false
t.index ["issue_id", "label_id"], name: "index_issues_labels_on_issue_id_and_label_id"
t.index ["label_id", "issue_id"], name: "index_issues_labels_on_label_id_and_issue_id"
t.index ["issue_id", "label_id"], name: "index_issues_labels_on_issue_id_and_label_id", using: :btree
t.index ["label_id", "issue_id"], name: "index_issues_labels_on_label_id_and_issue_id", using: :btree
end
create_table "issues_users", id: false, force: :cascade do |t|
create_table "issues_users", id: false, force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.integer "issue_id", null: false
t.integer "user_id", null: false
t.index ["issue_id", "user_id"], name: "index_issues_users_on_issue_id_and_user_id"
t.index ["user_id", "issue_id"], name: "index_issues_users_on_user_id_and_issue_id"
t.index ["issue_id", "user_id"], name: "index_issues_users_on_issue_id_and_user_id", using: :btree
t.index ["user_id", "issue_id"], name: "index_issues_users_on_user_id_and_issue_id", using: :btree
end
create_table "labels", force: :cascade do |t|
create_table "labels", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "url"
t.string "name"
t.string "color"
......@@ -66,10 +66,10 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.integer "repository_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["repository_id"], name: "index_labels_on_repository_id"
t.index ["repository_id"], name: "index_labels_on_repository_id", using: :btree
end
create_table "others", force: :cascade do |t|
create_table "others", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "data"
t.string "data1"
t.string "data2"
......@@ -78,7 +78,7 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "updated_at", null: false
end
create_table "repositories", force: :cascade do |t|
create_table "repositories", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "name"
t.string "full_name"
t.string "description"
......@@ -90,25 +90,25 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "updated_at", null: false
end
create_table "review_comments", force: :cascade do |t|
create_table "review_comments", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "url"
t.string "diff_hunk"
t.text "diff_hunk", limit: 65535
t.string "path"
t.string "body"
t.text "body", limit: 65535
t.string "html_url"
t.string "pull_request_url"
t.integer "issue_id"
t.integer "user_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.integer "rv_comment_id"
t.string "commit_id"
t.integer "score"
t.index ["issue_id"], name: "index_review_comments_on_issue_id"
t.index ["user_id"], name: "index_review_comments_on_user_id"
t.index ["issue_id"], name: "index_review_comments_on_issue_id", using: :btree
t.index ["user_id"], name: "index_review_comments_on_user_id", using: :btree
end
create_table "timelines", force: :cascade do |t|
create_table "timelines", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.integer "duration"
t.integer "issue_id"
t.string "label"
......@@ -116,10 +116,10 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "end_time"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["issue_id"], name: "index_timelines_on_issue_id"
t.index ["issue_id"], name: "index_timelines_on_issue_id", using: :btree
end
create_table "users", force: :cascade do |t|
create_table "users", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" do |t|
t.string "login"
t.string "url"
t.string "html_url"
......@@ -127,4 +127,12 @@ ActiveRecord::Schema.define(version: 20161230092006) do
t.datetime "updated_at", null: false
end
add_foreign_key "comments", "issues"
add_foreign_key "comments", "users"
add_foreign_key "issues", "repositories"
add_foreign_key "issues", "users"
add_foreign_key "labels", "repositories"
add_foreign_key "review_comments", "issues"
add_foreign_key "review_comments", "users"
add_foreign_key "timelines", "issues"
end
github.log
# Logfile created on 2016-12-02 13:00:03 +0700 by logger.rb/54362
I, [2017-01-04T15:00:03.289070 #26261] INFO -- : Begin task update []
I, [2017-01-04T15:00:03.424345 #26261] INFO -- : Insert issues
I, [2017-01-04T15:00:03.438357 #26261] INFO -- : Insert comments
I, [2017-01-04T15:00:03.438434 #26261] INFO -- : Size list_db_comment: 0
I, [2017-01-04T15:00:03.438459 #26261] INFO -- : Insert review comments
I, [2017-01-04T16:00:03.143334 #28901] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-04T17:00:02.964872 #31661] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-04T18:00:02.857221 #32584] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-04T19:00:03.619289 #968] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-04T20:00:03.369074 #1815] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-04T21:00:03.127965 #3054] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-04T22:00:02.932617 #3843] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-04T23:00:03.704021 #4560] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T00:00:03.429583 #5310] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T01:00:03.233233 #6061] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T02:00:02.997600 #6789] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T03:00:02.705295 #7538] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T04:00:03.489334 #8277] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T05:00:03.273519 #9034] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T06:00:02.979915 #9768] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T07:00:02.773562 #11108] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T08:00:03.649714 #12316] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T08:13:47.936594 #13113] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T08:14:36.072371 #13181] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T08:14:36.082629 #13181] INFO -- : Insert issues
I, [2017-01-05T08:25:11.215799 #13460] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T08:25:11.231126 #13460] INFO -- : Insert issues
I, [2017-01-05T08:26:32.623641 #13460] INFO -- : Insert comments
I, [2017-01-05T08:26:50.654178 #13460] INFO -- : Size list_db_comment: 351
I, [2017-01-05T08:26:50.654230 #13460] INFO -- : Insert review comments
I, [2017-01-05T08:27:41.441338 #13845] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T08:27:41.451404 #13845] INFO -- : Insert issues
I, [2017-01-05T08:28:59.141700 #13845] INFO -- : Insert comments
I, [2017-01-05T08:29:17.336594 #13845] INFO -- : Size list_db_comment: 351
I, [2017-01-05T08:29:17.336647 #13845] INFO -- : Insert review comments
I, [2017-01-05T08:29:35.885782 #13845] INFO -- : begin transaction
I, [2017-01-05T08:29:37.354456 #13845] INFO -- : Finished transaction in 1.468571352s
I, [2017-01-05T08:29:37.354510 #13845] INFO -- : Finished task update in 116.02315429s
I, [2017-01-05T08:52:20.571866 #31727] INFO -- : Begin task update socialtools
I, [2017-01-05T08:52:20.572338 #31727] INFO -- : Insert issues
I, [2017-01-05T08:52:34.506106 #31727] INFO -- : Insert comments
I, [2017-01-05T08:52:39.149096 #31727] INFO -- : Size list_db_comment: 81
I, [2017-01-05T08:52:39.149146 #31727] INFO -- : Insert review comments
I, [2017-01-05T08:52:41.312408 #31727] INFO -- : begin transaction
I, [2017-01-05T08:52:41.634016 #31727] INFO -- : Finished transaction in 0.321513519s
I, [2017-01-05T08:52:41.634100 #31727] INFO -- : Finished task update in 21.062234153s
I, [2017-01-05T09:00:03.424500 #15540] INFO -- : Begin task update ["socialtools", "arubaito-ex", "kuruma-ex"]
I, [2017-01-05T09:00:03.442879 #15540] INFO -- : Insert issues
I, [2017-01-05T09:01:27.824907 #15540] INFO -- : Insert comments
I, [2017-01-05T09:01:45.809676 #15540] INFO -- : Size list_db_comment: 351
I, [2017-01-05T09:01:45.809735 #15540] INFO -- : Insert review comments
I, [2017-01-05T09:02:04.084678 #15540] INFO -- : begin transaction
I, [2017-01-05T09:02:05.471034 #15540] INFO -- : Finished transaction in 1.386291218s
I, [2017-01-05T09:02:05.471401 #15540] INFO -- : Finished task update in 122.181024353s
I, [2017-01-05T09:21:51.947910 #31727] INFO -- : Begin task update socialtools
I, [2017-01-05T09:21:51.948319 #31727] INFO -- : Insert issues
I, [2017-01-05T09:22:07.174056 #31727] INFO -- : Insert comments
I, [2017-01-05T09:22:11.938348 #31727] INFO -- : Size list_db_comment: 81
I, [2017-01-05T09:22:11.938404 #31727] INFO -- : Insert review comments
I, [2017-01-05T09:22:14.149275 #31727] INFO -- : begin transaction
I, [2017-01-05T09:22:14.486619 #31727] INFO -- : Finished transaction in 0.337187306s
I, [2017-01-05T09:22:14.486681 #31727] INFO -- : Finished task update in 22.538768206s
I, [2017-01-05T09:38:53.299492 #17918] INFO -- : Begin task insert database from github
I, [2017-01-05T09:38:53.299591 #17918] INFO -- : Insert repos
I, [2017-01-05T09:38:59.590273 #17918] INFO -- : Insert users
I, [2017-01-05T09:39:05.411300 #17918] INFO -- : Insert labels
I, [2017-01-05T09:42:28.849940 #17918] INFO -- : Insert issues
I, [2017-01-05T09:49:27.770980 #17918] INFO -- : Insert comments
I, [2017-01-05T10:00:03.496559 #19630] INFO -- : Begin task update []
I, [2017-01-05T10:00:03.507098 #19630] INFO -- : Insert issues
I, [2017-01-05T10:00:03.508739 #19630] INFO -- : Insert comments
I, [2017-01-05T10:00:03.508799 #19630] INFO -- : Size list_db_comment: 0
I, [2017-01-05T10:00:03.508816 #19630] INFO -- : Insert review comments
I, [2017-01-05T10:00:03.508829 #19630] INFO -- : begin transaction
I, [2017-01-05T10:00:03.509340 #19630] INFO -- : Finished transaction in 0.000492336s
I, [2017-01-05T10:00:03.509370 #19630] INFO -- : Finished task update in 0.193458869s
I, [2017-01-05T10:00:45.905745 #19694] INFO -- : Begin task insert database from github
I, [2017-01-05T10:00:45.905915 #19694] INFO -- : Insert repos
I, [2017-01-05T10:00:52.540739 #19694] INFO -- : Insert users
I, [2017-01-05T10:00:58.119543 #19694] INFO -- : Insert labels
I, [2017-01-05T10:04:21.157281 #19694] INFO -- : Insert issues
I, [2017-01-05T10:11:21.982635 #19694] INFO -- : begin transaction 1
I, [2017-01-05T10:16:20.388899 #21356] INFO -- : Begin task insert database from github
I, [2017-01-05T10:16:20.388999 #21356] INFO -- : Insert repos
I, [2017-01-05T10:16:26.703851 #21356] INFO -- : Insert users
I, [2017-01-05T10:16:32.207170 #21356] INFO -- : Insert labels
I, [2017-01-05T10:20:02.422556 #21356] INFO -- : Insert issues
I, [2017-01-05T10:27:00.183270 #21356] INFO -- : begin transaction 1
I, [2017-01-05T10:58:10.694674 #24536] INFO -- : Begin task insert database from github
I, [2017-01-05T10:58:10.694818 #24536] INFO -- : Insert repos
I, [2017-01-05T10:58:17.094896 #24536] INFO -- : Insert users
I, [2017-01-05T10:58:22.728781 #24536] INFO -- : Insert labels
I, [2017-01-05T11:00:03.280450 #24851] INFO -- : Begin task update []
I, [2017-01-05T11:00:03.291063 #24851] INFO -- : Insert issues
I, [2017-01-05T11:00:03.292853 #24851] INFO -- : Insert comments
I, [2017-01-05T11:00:03.292923 #24851] INFO -- : Size list_db_comment: 0
I, [2017-01-05T11:00:03.292939 #24851] INFO -- : Insert review comments
I, [2017-01-05T11:00:03.292952 #24851] INFO -- : begin transaction
I, [2017-01-05T11:00:03.293399 #24851] INFO -- : Finished transaction in 0.00043004s
I, [2017-01-05T11:00:03.293428 #24851] INFO -- : Finished task update in 0.174704069s
I, [2017-01-05T11:01:45.480729 #24536] INFO -- : Insert issues
I, [2017-01-05T11:08:44.148734 #24536] INFO -- : begin transaction 1
I, [2017-01-05T11:09:39.575264 #26019] INFO -- : Begin task insert database from github
I, [2017-01-05T11:09:39.575395 #26019] INFO -- : Insert repos
I, [2017-01-05T11:09:46.130378 #26019] INFO -- : Insert users
I, [2017-01-05T11:09:51.677320 #26019] INFO -- : Insert labels
I, [2017-01-05T11:13:13.712508 #26019] INFO -- : begin transaction 1
I, [2017-01-05T11:13:16.314274 #26019] INFO -- : Finished transaction1 in 2.601687034s
I, [2017-01-05T11:13:16.314340 #26019] INFO -- : Insert comments
I, [2017-01-05T12:00:03.130245 #27472] INFO -- : Begin task update []
I, [2017-01-05T12:00:03.140000 #27472] INFO -- : Insert issues
I, [2017-01-05T12:00:03.142013 #27472] INFO -- : Insert comments
I, [2017-01-05T12:00:03.142111 #27472] INFO -- : Size list_db_comment: 0
I, [2017-01-05T12:00:03.142142 #27472] INFO -- : Insert review comments
I, [2017-01-05T12:00:03.142168 #27472] INFO -- : begin transaction
I, [2017-01-05T12:00:03.142917 #27472] INFO -- : Finished transaction in 0.000715033s
I, [2017-01-05T12:00:03.142980 #27472] INFO -- : Finished task update in 0.171218408s
I, [2017-01-05T12:39:21.212825 #28373] INFO -- : Begin task insert database from github
I, [2017-01-05T12:39:21.212924 #28373] INFO -- : Insert repos
I, [2017-01-05T12:39:27.749398 #28373] INFO -- : Insert users
I, [2017-01-05T12:39:33.416002 #28373] INFO -- : Insert labels
I, [2017-01-05T12:44:40.053934 #28373] INFO -- : begin transaction 1
I, [2017-01-05T12:44:42.796909 #28373] INFO -- : Finished transaction1 in 2.742796386s
I, [2017-01-05T12:44:42.796973 #28373] INFO -- : Finished transaction2 in 2.742863166s
I, [2017-01-05T12:44:42.797012 #28373] INFO -- : Finished task import in 321.584075235s
I, [2017-01-05T12:58:31.198332 #29698] INFO -- : Begin task insert database from github
I, [2017-01-05T12:58:31.198434 #29698] INFO -- : Insert repos
I, [2017-01-05T12:58:44.112325 #29698] INFO -- : Insert users
I, [2017-01-05T12:58:56.093587 #29698] INFO -- : Insert labels
I, [2017-01-05T13:00:02.979954 #29819] INFO -- : Begin task update []
I, [2017-01-05T13:00:02.990775 #29819] INFO -- : Insert issues
I, [2017-01-05T13:00:02.992515 #29819] INFO -- : Insert comments
I, [2017-01-05T13:00:02.992582 #29819] INFO -- : Size list_db_comment: 0
I, [2017-01-05T13:00:02.992615 #29819] INFO -- : Insert review comments
I, [2017-01-05T13:00:02.992634 #29819] INFO -- : begin transaction
I, [2017-01-05T13:00:02.993092 #29819] INFO -- : Finished transaction in 0.000437697s
I, [2017-01-05T13:00:02.993124 #29819] INFO -- : Finished task update in 0.179208255s
I, [2017-01-05T13:00:50.005286 #29890] INFO -- : Begin task insert database from github
I, [2017-01-05T13:00:50.005411 #29890] INFO -- : Insert repos
I, [2017-01-05T13:00:56.559168 #29890] INFO -- : Insert users
I, [2017-01-05T13:01:29.046913 #30033] INFO -- : Begin task insert database from github
I, [2017-01-05T13:01:29.047006 #30033] INFO -- : Insert repos
I, [2017-01-05T13:01:35.451138 #30033] INFO -- : Insert users
I, [2017-01-05T13:01:41.102358 #30033] INFO -- : Insert labels
I, [2017-01-05T13:06:26.761286 #30033] INFO -- : Finished task import in 297.714270051s
I, [2017-01-05T13:20:05.697642 #31461] INFO -- : Begin task insert database from github
I, [2017-01-05T13:20:05.697744 #31461] INFO -- : Insert repos
I, [2017-01-05T13:20:18.022428 #31461] INFO -- : Insert users
I, [2017-01-05T13:20:29.450854 #31461] INFO -- : Insert labels
I, [2017-01-05T13:26:45.478394 #31461] INFO -- : Insert issues
I, [2017-01-05T13:43:15.727652 #31461] INFO -- : Finished task import in 1390.029902092s
I, [2017-01-05T13:56:56.944997 #3462] INFO -- : Begin task insert comments
I, [2017-01-05T13:56:56.945132 #3462] INFO -- : Insert comments
I, [2017-01-05T14:00:02.924944 #4006] INFO -- : Begin task update []
I, [2017-01-05T14:05:37.695034 #3462] INFO -- : Insert review comments
I, [2017-01-05T14:15:17.310927 #5234] INFO -- : Begin task insert comments
I, [2017-01-05T14:15:17.311031 #5234] INFO -- : Insert comments
I, [2017-01-05T14:22:06.169287 #5234] INFO -- : Insert review comments
I, [2017-01-05T14:27:10.490299 #6854] INFO -- : Begin task insert comments
I, [2017-01-05T14:27:10.490449 #6854] INFO -- : Insert comments
I, [2017-01-05T14:34:45.327430 #6854] INFO -- : Insert review comments
I, [2017-01-05T14:42:30.739559 #8845] INFO -- : Begin task insert comments
I, [2017-01-05T14:42:30.739654 #8845] INFO -- : Insert comments
I, [2017-01-05T14:47:56.322951 #8845] INFO -- : Insert review comments
I, [2017-01-05T14:59:59.083592 #11018] INFO -- : Begin task insert database from github
I, [2017-01-05T14:59:59.083684 #11018] INFO -- : Insert repos
I, [2017-01-05T15:00:04.041227 #11108] INFO -- : Begin task update []
I, [2017-01-05T15:00:12.018425 #11018] INFO -- : Insert users
I, [2017-01-05T15:00:23.279065 #11018] INFO -- : Insert labels
I, [2017-01-05T15:06:09.861693 #11018] INFO -- : Insert issues
I, [2017-01-05T15:18:05.421238 #11018] INFO -- : Finished task import in 1086.337545379s
I, [2017-01-05T15:19:52.765313 #12922] INFO -- : Begin task insert comments
I, [2017-01-05T15:19:52.765411 #12922] INFO -- : Insert comments
I, [2017-01-05T15:26:14.109653 #12922] INFO -- : Insert review comments
I, [2017-01-05T15:34:30.943277 #12922] INFO -- : Finished task import comments in 878.177847578s
I, [2017-01-05T16:00:03.679176 #17006] INFO -- : Begin task update []
lib/github_loader.rb
......@@ -11,7 +11,7 @@ class GithubLoader
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
list_db_repo = []
# list_db_repo = []
for i in (1..last_page)
repos = client.org_repos($org, {type: 'all', page: i})
repos.each do |repo|
......@@ -29,15 +29,17 @@ class GithubLoader
db_repo.updated_at = repo.updated_at
end
list_db_repo.append(db_repo)
db_repo.save
# list_db_repo.append(db_repo)
end
end
list_db_repo
# list_db_repo
end
def insert_labels(client, db_repos)
def insert_labels(client)
$logger.info "Insert labels"
list_db_label = []
# list_db_label = []
db_repos = Repository.all
db_repos.each do |repo|
labels = client.labels($org + "/" + repo.name)
last_page = 1
......@@ -48,22 +50,25 @@ class GithubLoader
for i in (1..last_page)
labels = client.labels($org + "/" + repo.name, {page: i})
labels.each do |label|
db_label = Label.find_by(id: label.id)
normalized_label = label.name.gsub(/[^0-9A-Za-z]/, "").downcase # normalize string
db_label = Label.find_by(name: normalized_label)
if db_label.nil?
db_label = Label.new
db_label.id = label.id
db_label.url = label.url
db_label.repository_id = repo.id
end
db_label.name = label.name.gsub(/[^0-9A-Za-z]/, "").downcase # normalize string
db_label.name = normalized_label
db_label.color = label.color
db_label.default = label.default
list_db_label.append(db_label)
db_label.save
# list_db_label.append(db_label)
end
end
end
list_db_label
# list_db_label
end
def insert_users(client)
......@@ -74,7 +79,7 @@ class GithubLoader
last_page = client.last_response.headers[:link].split(' ')[2][-3].to_i
end
list_db_user = []
# list_db_user = []
for i in (1..last_page)
users = client.org_members($org, {page: i})
users.each do |user|
......@@ -87,15 +92,19 @@ class GithubLoader
db_user.html_url = user.html_url
end
list_db_user.append(db_user)
db_user.save
# list_db_user.append(db_user)
end
end
list_db_user
# list_db_user
end
def insert_issues(client, list_db_user, list_db_label, db_repos)
def insert_issues(client)
$logger.info "Insert issues"
list_db_issue = []
list_db_user = User.all
list_db_label = Label.all
db_repos = Repository.all
# list_db_issue = []
db_repos.each do |repo|
issues = client.list_issues($org + "/" + repo.name, { state: "all" })
......@@ -116,6 +125,9 @@ class GithubLoader
db_issue.number = issue.number
db_issue.repository_id = repo.id
db_issue.user_id = issue.user.id
if User.find_by(id: issue.user.id).nil?
next
end
db_issue.created_at = issue.created_at
db_issue.html_url = issue.html_url
if issue[:pull_request].nil?
......@@ -136,8 +148,9 @@ class GithubLoader
old_label = db_issue.labels.find_by(name: col_labels)
db_issue.labels.clear
issue.labels.each do |label|
label_name = label.name.gsub(/[^0-9A-Za-z]/, "").downcase
list_db_label.each do |db_label|
if db_label.id == label.id
if db_label.name == label_name
db_issue.labels.append(db_label)
break
end
......@@ -150,8 +163,12 @@ class GithubLoader
end
elsif new_label.name != old_label.name
timeline = Timeline.where(issue_id: db_issue.id, label: old_label.name).first
timeline.end_time = Time.now
timeline.save
begin
timeline.end_time = Time.now
timeline.save
rescue
byebug
end
new_timeline(db_issue.id, new_label.name)
elsif new_label.name == old_label.name
......@@ -172,16 +189,19 @@ class GithubLoader
end
end
list_db_issue.append(db_issue)
db_issue.save
# list_db_issue.append(db_issue)
end
end
end
list_db_issue
# list_db_issue
end
def insert_comments(client, list_db_repo)
def insert_comments(client)
$logger.info "Insert comments"
list_db_comment = []
# list_db_comment = []
list_db_repo = Repository.all
list_db_repo.each do |db_repo|
comments = client.issues_comments($org + "/" + db_repo.name)
......@@ -207,17 +227,19 @@ class GithubLoader
db_comment.user_id = comment.user.id
end
db_comment.body = comment.body
list_db_comment.append(db_comment)
db_comment.save
# list_db_comment.append(db_comment)
end
end
end
$logger.info "Size list_db_comment: " + list_db_comment.size.to_s
list_db_comment
# $logger.info "Size list_db_comment: " + list_db_comment.size.to_s
# list_db_comment
end
def insert_review_comments(client, list_db_repo)
def insert_review_comments(client)
$logger.info "Insert review comments"
list_db_rv_comment = []
# list_db_rv_comment = []
list_db_repo = Repository.all
list_db_repo.each do |db_repo|
rv_comments = client.pull_requests_comments($org + "/" + db_repo.name)
last_page = 1
......@@ -241,6 +263,9 @@ class GithubLoader
db_rv_comment.created_at = rv_comment.created_at
unless rv_comment.user.nil?
db_rv_comment.user_id = rv_comment.user.id
if User.find_by(id: rv_comment.user.id).nil?
next
end
end
pull_request_url_split = rv_comment.pull_request_url.split("/")
......@@ -252,11 +277,12 @@ class GithubLoader
db_rv_comment.updated_at = rv_comment.updated_at
db_rv_comment.body = rv_comment.body
list_db_rv_comment.append(db_rv_comment)
db_rv_comment.save
# list_db_rv_comment.append(db_rv_comment)
end
end
end
list_db_rv_comment
# list_db_rv_comment
end
def update_issue_pull_association
......@@ -277,12 +303,6 @@ class GithubLoader
end
end
# def update_repo(repositories)
# repositories.each do |repo|
# end
# end
def new_timeline(issue_id, label)
timeline = Timeline.new
timeline.issue_id = issue_id
......
lib/tasks/github.rake
......@@ -16,42 +16,49 @@ namespace :github do
$logger.info "Begin task insert database from github"
start_time = Time.now
list_db_repo = loader.insert_repos($client)
list_db_user = loader.insert_users($client)
list_db_label = loader.insert_labels($client, list_db_repo)
list_db_issue = loader.insert_issues($client, list_db_user, list_db_label, list_db_repo)
list_db_comment = loader.insert_comments($client, list_db_repo)
list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
loader.insert_repos($client)
loader.insert_users($client)
loader.insert_labels($client)
loader.insert_issues($client)
$logger.info "begin transaction"
start_time_transaction = Time.now
ActiveRecord::Base.transaction do
loader.update_issue_pull_association
list_db_repo.each do |db_repo|
db_repo.save
end
# $logger.info "begin transaction 1"
# start_time_transaction = Time.now
# ActiveRecord::Base.transaction do
list_db_user.each do |db_user|
db_user.save
end
# list_db_repo.each do |db_repo|
# db_repo.save
# end
list_db_label.each do |db_label|
db_label.save
end
# list_db_user.each do |db_user|
# db_user.save
# end
list_db_issue.each do |db_issue|
db_issue.save
end
# list_db_label.each do |db_label|
# db_label.save
# end
list_db_comment.each do |db_comment|
db_comment.save
end
list_db_rv_comment.each do |db_rv_comment|
db_rv_comment.save
end
end
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
# list_db_issue.each do |db_issue|
# db_issue.save
# end
# end
# $logger.info "Finished transaction1 in #{Time.now - start_time_transaction}s"
# list_db_comment = loader.insert_comments($client, list_db_repo)
# list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
# start_time_transaction = Time.now
# ActiveRecord::Base.transaction do
# list_db_comment.each do |db_comment|
# db_comment.save
# end
# list_db_rv_comment.each do |db_rv_comment|
# db_rv_comment.save
# end
# end
# $logger.info "Finished transaction2 in #{Time.now - start_time_transaction}s"
$logger.info "Finished task import in #{Time.now - start_time}s"
end
......@@ -59,15 +66,16 @@ namespace :github do
task insert_github_data_for_specific_repos: :environment do
loader = GithubLoader.new
start_time = Time.now
$logger.info "Begin task update " + $repos_to_update.to_s
list_db_repo = Repository.where(name: $repos_to_update)
repos_to_update = Other.where(data_type: 2).map{ |x|x.data }
$logger.info "Begin task update " + repos_to_update.to_s
list_db_repo = Repository.where(name: repos_to_update)
list_db_user = User.all
list_db_label = Label.all
list_db_issue = loader.insert_issues($client, list_db_user, list_db_label, list_db_repo)
list_db_comment = loader.insert_comments($client, list_db_repo)
list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
logger.info "begin transaction"
$logger.info "begin transaction"
start_time_transaction = Time.now
ActiveRecord::Base.transaction do
......@@ -83,8 +91,8 @@ namespace :github do
db_rv_comment.save
end
end
logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
logger.info "Finished task update in #{Time.now - start_time}s"
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
$logger.info "Finished task update in #{Time.now - start_time}s"
end
desc "Insert Repository only"
......@@ -100,10 +108,6 @@ namespace :github do
db_repo.save
end
# list_db_user.each do |db_user|
# db_user.save
# end
list_db_label.each do |db_label|
db_label.save
end
......@@ -116,21 +120,20 @@ namespace :github do
loader = GithubLoader.new
$logger.info "Begin task insert comments"
start_time = Time.now
list_db_repo = Repository.all
list_db_comment = loader.insert_comments($client, list_db_repo)
list_db_rv_comment = loader.insert_review_comments($client, list_db_repo)
start_time_transaction = Time.now
ActiveRecord::Base.transaction do
list_db_comment.each do |db_comment|
db_comment.save
end
list_db_rv_comment.each do |db_rv_comment|
db_rv_comment.save
end
end
$logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
loader.insert_comments($client)
loader.insert_review_comments($client)
# start_time_transaction = Time.now
# ActiveRecord::Base.transaction do
# list_db_comment.each do |db_comment|
# db_comment.save
# end
# list_db_rv_comment.each do |db_rv_comment|
# db_rv_comment.save
# end
# end
# $logger.info "Finished transaction in #{Time.now - start_time_transaction}s"
$logger.info "Finished task import comments in #{Time.now - start_time}s"
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment