mirror of
https://github.com/SrIzan10/hc-harbor.git
synced 2026-05-01 10:45:21 +00:00
Sync in and display repo metadata
This commit is contained in:
@@ -11,6 +11,9 @@
|
|||||||
padding: 1rem;
|
padding: 1rem;
|
||||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
|
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
|
||||||
transition: transform 0.2s ease;
|
transition: transform 0.2s ease;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.5rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.project-duration-card:hover {
|
.project-duration-card:hover {
|
||||||
@@ -21,8 +24,21 @@
|
|||||||
.project-header {
|
.project-header {
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
|
align-items: flex-start;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.project-name-section {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.25rem;
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.project-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: 0.5rem;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.project-header strong {
|
.project-header strong {
|
||||||
@@ -33,6 +49,34 @@
|
|||||||
.project-time {
|
.project-time {
|
||||||
font-size: 0.875rem;
|
font-size: 0.875rem;
|
||||||
color: var(--muted-color);
|
color: var(--muted-color);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.project-stars {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
color: var(--muted-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
.project-description {
|
||||||
|
font-size: 0.875rem;
|
||||||
|
color: var(--muted-color);
|
||||||
|
line-height: 1.4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.project-languages {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
color: var(--muted-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
.languages-label {
|
||||||
|
font-weight: 600;
|
||||||
|
margin-right: 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.project-meta {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
color: var(--muted-color);
|
||||||
|
margin-top: 0.25rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.project-progress-bar {
|
.project-progress-bar {
|
||||||
|
|||||||
@@ -95,7 +95,7 @@ class StaticPagesController < ApplicationController
|
|||||||
def project_durations
|
def project_durations
|
||||||
return unless current_user
|
return unless current_user
|
||||||
|
|
||||||
@project_repo_mappings = current_user.project_repo_mappings
|
@project_repo_mappings = current_user.project_repo_mappings.includes(:repository)
|
||||||
cache_key = "user_#{current_user.id}_project_durations_#{params[:interval]}"
|
cache_key = "user_#{current_user.id}_project_durations_#{params[:interval]}"
|
||||||
cache_key += "_#{params[:from]}_#{params[:to]}" if params[:interval] == "custom"
|
cache_key += "_#{params[:from]}_#{params[:to]}" if params[:interval] == "custom"
|
||||||
|
|
||||||
@@ -104,9 +104,11 @@ class StaticPagesController < ApplicationController
|
|||||||
project_times = heartbeats.group(:project).duration_seconds
|
project_times = heartbeats.group(:project).duration_seconds
|
||||||
project_labels = current_user.project_labels
|
project_labels = current_user.project_labels
|
||||||
project_times.map do |project, duration|
|
project_times.map do |project, duration|
|
||||||
|
mapping = @project_repo_mappings.find { |p| p.project_name == project }
|
||||||
{
|
{
|
||||||
project: project_labels.find { |p| p.project_key == project }&.label || project || "Unknown",
|
project: project_labels.find { |p| p.project_key == project }&.label || project || "Unknown",
|
||||||
repo_url: @project_repo_mappings.find { |p| p.project_name == project }&.repo_url,
|
repo_url: mapping&.repo_url,
|
||||||
|
repository: mapping&.repository,
|
||||||
duration: duration
|
duration: duration
|
||||||
}
|
}
|
||||||
end.filter { |p| p[:duration].positive? }.sort_by { |p| p[:duration] }.reverse
|
end.filter { |p| p[:duration].positive? }.sort_by { |p| p[:duration] }.reverse
|
||||||
|
|||||||
@@ -10,9 +10,10 @@ class ProcessCommitJob < ApplicationJob
|
|||||||
|
|
||||||
discard_on ActiveJob::DeserializationError # If User record is gone
|
discard_on ActiveJob::DeserializationError # If User record is gone
|
||||||
|
|
||||||
def perform(user_id, commit_sha, commit_api_url, provider_string)
|
def perform(user_id, commit_sha, commit_api_url, provider_string, repository_id = nil)
|
||||||
provider_sym = provider_string.to_sym # Convert string back to symbol
|
provider_sym = provider_string.to_sym # Convert string back to symbol
|
||||||
user = User.find_by(id: user_id)
|
user = User.find_by(id: user_id)
|
||||||
|
repository = repository_id ? Repository.find_by(id: repository_id) : nil
|
||||||
|
|
||||||
unless user
|
unless user
|
||||||
Rails.logger.warn "[ProcessCommitJob] User ##{user_id} not found. Skipping commit #{commit_sha}."
|
Rails.logger.warn "[ProcessCommitJob] User ##{user_id} not found. Skipping commit #{commit_sha}."
|
||||||
@@ -31,10 +32,10 @@ class ProcessCommitJob < ApplicationJob
|
|||||||
|
|
||||||
case provider_sym
|
case provider_sym
|
||||||
when :github
|
when :github
|
||||||
process_github_commit(user, commit_sha, commit_api_url)
|
process_github_commit(user, commit_sha, commit_api_url, repository)
|
||||||
# Add other providers like :gitlab later
|
# Add other providers like :gitlab later
|
||||||
# when :gitlab
|
# when :gitlab
|
||||||
# process_gitlab_commit(user, commit_sha, commit_api_url)
|
# process_gitlab_commit(user, commit_sha, commit_api_url, repository)
|
||||||
else
|
else
|
||||||
Rails.logger.error "[ProcessCommitJob] Unknown provider '#{provider_sym}' for commit #{commit_sha}."
|
Rails.logger.error "[ProcessCommitJob] Unknown provider '#{provider_sym}' for commit #{commit_sha}."
|
||||||
end
|
end
|
||||||
@@ -42,7 +43,7 @@ class ProcessCommitJob < ApplicationJob
|
|||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def process_github_commit(user, commit_sha, commit_api_url)
|
def process_github_commit(user, commit_sha, commit_api_url, repository)
|
||||||
unless user.github_access_token.present?
|
unless user.github_access_token.present?
|
||||||
Rails.logger.warn "[ProcessCommitJob] User ##{user.id} missing GitHub token for commit #{commit_sha}. Skipping."
|
Rails.logger.warn "[ProcessCommitJob] User ##{user.id} missing GitHub token for commit #{commit_sha}. Skipping."
|
||||||
return
|
return
|
||||||
@@ -82,6 +83,7 @@ class ProcessCommitJob < ApplicationJob
|
|||||||
Commit.create!(
|
Commit.create!(
|
||||||
sha: api_commit_sha,
|
sha: api_commit_sha,
|
||||||
user_id: user.id,
|
user_id: user.id,
|
||||||
|
repository_id: repository&.id,
|
||||||
github_raw: commit_data_json,
|
github_raw: commit_data_json,
|
||||||
created_at: commit_actual_created_at, # Manually set created_at
|
created_at: commit_actual_created_at, # Manually set created_at
|
||||||
updated_at: Time.current # Let Rails handle updated_at, or set explicitly
|
updated_at: Time.current # Let Rails handle updated_at, or set explicitly
|
||||||
@@ -95,7 +97,7 @@ class ProcessCommitJob < ApplicationJob
|
|||||||
reset_time = Time.at(response.headers["X-RateLimit-Reset"].to_i)
|
reset_time = Time.at(response.headers["X-RateLimit-Reset"].to_i)
|
||||||
delay_seconds = [ (reset_time - Time.current).ceil, 5 ].max # at least 5s delay
|
delay_seconds = [ (reset_time - Time.current).ceil, 5 ].max # at least 5s delay
|
||||||
Rails.logger.warn "[ProcessCommitJob] GitHub API rate limit exceeded for User ##{user.id}. Retrying in #{delay_seconds}s. URL: #{commit_api_url}"
|
Rails.logger.warn "[ProcessCommitJob] GitHub API rate limit exceeded for User ##{user.id}. Retrying in #{delay_seconds}s. URL: #{commit_api_url}"
|
||||||
self.class.set(wait: delay_seconds.seconds).perform_later(user.id, commit_sha, commit_api_url, "github")
|
self.class.set(wait: delay_seconds.seconds).perform_later(user.id, commit_sha, commit_api_url, "github", repository&.id)
|
||||||
else
|
else
|
||||||
Rails.logger.error "[ProcessCommitJob] GitHub API forbidden (403) for User ##{user.id}. URL: #{commit_api_url}. Response: #{response.body.to_s.truncate(500)}"
|
Rails.logger.error "[ProcessCommitJob] GitHub API forbidden (403) for User ##{user.id}. URL: #{commit_api_url}. Response: #{response.body.to_s.truncate(500)}"
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -22,7 +22,11 @@ class PullRepoCommitsJob < ApplicationJob
|
|||||||
return
|
return
|
||||||
end
|
end
|
||||||
|
|
||||||
Rails.logger.info "[PullRepoCommitsJob] Pulling commits for #{owner}/#{repo} for User ##{user.id}"
|
# Find the repository record
|
||||||
|
repo_url = "https://github.com/#{owner}/#{repo}"
|
||||||
|
repository = Repository.find_by(url: repo_url)
|
||||||
|
|
||||||
|
Rails.logger.info "[PullRepoCommitsJob] Pulling commits for #{owner}/#{repo} for User ##{user.id} (Repository: #{repository&.id})"
|
||||||
|
|
||||||
# Get commits from the last 3 days
|
# Get commits from the last 3 days
|
||||||
since_date = 3.days.ago.iso8601
|
since_date = 3.days.ago.iso8601
|
||||||
@@ -37,7 +41,7 @@ class PullRepoCommitsJob < ApplicationJob
|
|||||||
|
|
||||||
if response.status.success?
|
if response.status.success?
|
||||||
commits_data = response.parse
|
commits_data = response.parse
|
||||||
process_commits(user, commits_data)
|
process_commits(user, commits_data, repository)
|
||||||
elsif response.status.code == 404
|
elsif response.status.code == 404
|
||||||
Rails.logger.warn "[PullRepoCommitsJob] Repository #{owner}/#{repo} not found (404) for User ##{user.id}."
|
Rails.logger.warn "[PullRepoCommitsJob] Repository #{owner}/#{repo} not found (404) for User ##{user.id}."
|
||||||
elsif response.status.code == 403 # Forbidden, could be rate limit or permissions
|
elsif response.status.code == 403 # Forbidden, could be rate limit or permissions
|
||||||
@@ -65,7 +69,7 @@ class PullRepoCommitsJob < ApplicationJob
|
|||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def process_commits(user, commits_data)
|
def process_commits(user, commits_data, repository)
|
||||||
return if commits_data.empty?
|
return if commits_data.empty?
|
||||||
|
|
||||||
# Get existing commit SHAs to avoid duplicates
|
# Get existing commit SHAs to avoid duplicates
|
||||||
@@ -106,7 +110,8 @@ class PullRepoCommitsJob < ApplicationJob
|
|||||||
user.id,
|
user.id,
|
||||||
commit_sha,
|
commit_sha,
|
||||||
commit_api_url,
|
commit_api_url,
|
||||||
"github"
|
"github",
|
||||||
|
repository&.id
|
||||||
)
|
)
|
||||||
enqueued_count += 1
|
enqueued_count += 1
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -53,11 +53,23 @@ class ScanRepoEventsForCommitsJob < ApplicationJob
|
|||||||
next
|
next
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Extract repository info from commit API URL
|
||||||
|
# Format: https://api.github.com/repos/owner/repo/commits/sha
|
||||||
|
repository_id = nil
|
||||||
|
if commit_api_url =~ %r{https://api\.github\.com/repos/([^/]+)/([^/]+)/commits/}
|
||||||
|
owner = $1
|
||||||
|
repo = $2
|
||||||
|
repo_url = "https://github.com/#{owner}/#{repo}"
|
||||||
|
repository = Repository.find_by(url: repo_url)
|
||||||
|
repository_id = repository&.id
|
||||||
|
end
|
||||||
|
|
||||||
potential_commits_buffer << {
|
potential_commits_buffer << {
|
||||||
sha: commit_sha,
|
sha: commit_sha,
|
||||||
api_url: commit_api_url,
|
api_url: commit_api_url,
|
||||||
user_id: user.id,
|
user_id: user.id,
|
||||||
provider: event.provider.to_s
|
provider: event.provider.to_s,
|
||||||
|
repository_id: repository_id
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -101,7 +113,8 @@ class ScanRepoEventsForCommitsJob < ApplicationJob
|
|||||||
commit_details[:user_id],
|
commit_details[:user_id],
|
||||||
commit_details[:sha],
|
commit_details[:sha],
|
||||||
commit_details[:api_url],
|
commit_details[:api_url],
|
||||||
commit_details[:provider]
|
commit_details[:provider],
|
||||||
|
commit_details[:repository_id]
|
||||||
)
|
)
|
||||||
enqueued_count += 1
|
enqueued_count += 1
|
||||||
end
|
end
|
||||||
|
|||||||
36
app/jobs/sync_repo_metadata_job.rb
Normal file
36
app/jobs/sync_repo_metadata_job.rb
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
class SyncRepoMetadataJob < ApplicationJob
|
||||||
|
queue_as :default
|
||||||
|
|
||||||
|
retry_on HTTP::TimeoutError, HTTP::ConnectionError, wait: :exponentially_longer, attempts: 3
|
||||||
|
retry_on JSON::ParserError, wait: 10.seconds, attempts: 2
|
||||||
|
discard_on ArgumentError # Invalid repository URLs
|
||||||
|
|
||||||
|
def perform(repository_id)
|
||||||
|
repository = Repository.find_by(id: repository_id)
|
||||||
|
return unless repository
|
||||||
|
|
||||||
|
Rails.logger.info "[SyncRepoMetadataJob] Syncing metadata for #{repository.url}"
|
||||||
|
|
||||||
|
begin
|
||||||
|
# Use any user who has mapped to this repository for API access
|
||||||
|
user = repository.users.joins(:project_repo_mappings).first
|
||||||
|
return unless user
|
||||||
|
|
||||||
|
service = RepoHost::ServiceFactory.for_url(user, repository.url)
|
||||||
|
metadata = service.fetch_repo_metadata
|
||||||
|
|
||||||
|
if metadata
|
||||||
|
repository.update!(metadata)
|
||||||
|
Rails.logger.info "[SyncRepoMetadataJob] Updated metadata for #{repository.url}"
|
||||||
|
else
|
||||||
|
Rails.logger.warn "[SyncRepoMetadataJob] No metadata returned for #{repository.url}"
|
||||||
|
end
|
||||||
|
rescue ArgumentError => e
|
||||||
|
Rails.logger.error "[SyncRepoMetadataJob] #{e.message} for repository #{repository.id}"
|
||||||
|
raise # Discard job for unsupported hosts
|
||||||
|
rescue => e
|
||||||
|
Rails.logger.error "[SyncRepoMetadataJob] Unexpected error: #{e.message}"
|
||||||
|
raise # Retry for other errors
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
53
app/jobs/sync_stale_repo_metadata_job.rb
Normal file
53
app/jobs/sync_stale_repo_metadata_job.rb
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
class SyncStaleRepoMetadataJob < ApplicationJob
|
||||||
|
queue_as :default
|
||||||
|
|
||||||
|
def perform
|
||||||
|
Rails.logger.info "[SyncStaleRepoMetadataJob] Starting sync of stale repository metadata"
|
||||||
|
|
||||||
|
# Find all mappings where the repository has stale metadata or is missing metadata entirely
|
||||||
|
mappings_with_stale_repos = ProjectRepoMapping.includes(:repository, :user)
|
||||||
|
.joins(:repository)
|
||||||
|
.where("repositories.last_synced_at IS NULL OR repositories.last_synced_at < ?", 1.day.ago)
|
||||||
|
|
||||||
|
# Also find mappings where repository is nil (shouldn't happen, but just in case)
|
||||||
|
mappings_without_repos = ProjectRepoMapping.includes(:user)
|
||||||
|
.where(repository: nil)
|
||||||
|
|
||||||
|
all_stale_mappings = mappings_with_stale_repos.to_a + mappings_without_repos.to_a
|
||||||
|
|
||||||
|
Rails.logger.info "[SyncStaleRepoMetadataJob] Found #{all_stale_mappings.count} project mappings with stale or missing repository metadata"
|
||||||
|
|
||||||
|
# Group by repository to avoid duplicate API calls
|
||||||
|
repos_to_sync = {}
|
||||||
|
|
||||||
|
all_stale_mappings.each do |mapping|
|
||||||
|
if mapping.repository
|
||||||
|
repos_to_sync[mapping.repository.id] = mapping.repository
|
||||||
|
else
|
||||||
|
# Handle mappings without repository - recreate the repository
|
||||||
|
Rails.logger.warn "[SyncStaleRepoMetadataJob] Found mapping without repository: #{mapping.inspect}"
|
||||||
|
if mapping.repo_url.present?
|
||||||
|
begin
|
||||||
|
repo = Repository.find_or_create_by_url(mapping.repo_url)
|
||||||
|
mapping.update!(repository: repo)
|
||||||
|
repos_to_sync[repo.id] = repo
|
||||||
|
rescue => e
|
||||||
|
Rails.logger.error "[SyncStaleRepoMetadataJob] Failed to create repository for mapping #{mapping.id}: #{e.message}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
Rails.logger.info "[SyncStaleRepoMetadataJob] Enqueuing sync for #{repos_to_sync.count} unique repositories"
|
||||||
|
|
||||||
|
repos_to_sync.each_value do |repository|
|
||||||
|
# Only sync if the repository has at least one user (needed for API access)
|
||||||
|
next unless repository.users.exists?
|
||||||
|
|
||||||
|
Rails.logger.info "[SyncStaleRepoMetadataJob] Enqueuing sync for #{repository.url}"
|
||||||
|
SyncRepoMetadataJob.perform_later(repository.id)
|
||||||
|
end
|
||||||
|
|
||||||
|
Rails.logger.info "[SyncStaleRepoMetadataJob] Completed enqueuing sync jobs"
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -4,6 +4,7 @@ class Commit < ApplicationRecord
|
|||||||
self.primary_key = :sha
|
self.primary_key = :sha
|
||||||
|
|
||||||
belongs_to :user
|
belongs_to :user
|
||||||
|
belongs_to :repository, optional: true
|
||||||
|
|
||||||
validates :sha, presence: true, uniqueness: true
|
validates :sha, presence: true, uniqueness: true
|
||||||
validates :user_id, presence: true
|
validates :user_id, presence: true
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
class ProjectRepoMapping < ApplicationRecord
|
class ProjectRepoMapping < ApplicationRecord
|
||||||
belongs_to :user
|
belongs_to :user
|
||||||
|
belongs_to :repository, optional: true
|
||||||
|
|
||||||
has_paper_trail
|
has_paper_trail
|
||||||
|
|
||||||
@@ -20,7 +21,8 @@ class ProjectRepoMapping < ApplicationRecord
|
|||||||
"<<LAST PROJECT>>"
|
"<<LAST PROJECT>>"
|
||||||
]
|
]
|
||||||
|
|
||||||
after_create :schedule_commit_pull
|
after_create :create_repository_and_sync
|
||||||
|
after_update :sync_repository_if_url_changed
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
@@ -30,6 +32,24 @@ class ProjectRepoMapping < ApplicationRecord
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def create_repository_and_sync
|
||||||
|
# Create or find repository record
|
||||||
|
repo = Repository.find_or_create_by_url(repo_url)
|
||||||
|
update_column(:repository_id, repo.id)
|
||||||
|
|
||||||
|
# Schedule commit pull and metadata sync
|
||||||
|
schedule_commit_pull
|
||||||
|
SyncRepoMetadataJob.perform_later(repo.id)
|
||||||
|
end
|
||||||
|
|
||||||
|
def sync_repository_if_url_changed
|
||||||
|
if saved_change_to_repo_url?
|
||||||
|
repo = Repository.find_or_create_by_url(repo_url)
|
||||||
|
update_column(:repository_id, repo.id)
|
||||||
|
SyncRepoMetadataJob.perform_later(repo.id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def schedule_commit_pull
|
def schedule_commit_pull
|
||||||
# Extract owner and repo name from the URL
|
# Extract owner and repo name from the URL
|
||||||
# Example URL: https://github.com/owner/repo
|
# Example URL: https://github.com/owner/repo
|
||||||
|
|||||||
44
app/models/repository.rb
Normal file
44
app/models/repository.rb
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
class Repository < ApplicationRecord
|
||||||
|
has_many :project_repo_mappings, dependent: :destroy
|
||||||
|
has_many :users, through: :project_repo_mappings
|
||||||
|
has_many :commits, dependent: :destroy
|
||||||
|
|
||||||
|
validates :url, presence: true, uniqueness: true
|
||||||
|
validates :host, presence: true
|
||||||
|
validates :owner, presence: true
|
||||||
|
validates :name, presence: true
|
||||||
|
|
||||||
|
# Check if metadata needs refreshing (older than 1 day)
|
||||||
|
def metadata_stale?
|
||||||
|
last_synced_at.nil? || last_synced_at < 1.day.ago
|
||||||
|
end
|
||||||
|
|
||||||
|
# Get formatted languages list
|
||||||
|
def formatted_languages
|
||||||
|
return nil if languages.blank?
|
||||||
|
languages.split(", ").first(3).join(", ") + (languages.split(", ").length > 3 ? "..." : "")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Parse owner and repo from URL
|
||||||
|
def self.parse_url(url)
|
||||||
|
if url =~ %r{https?://([^/]+)/([^/]+)/([^/]+)/?$}
|
||||||
|
{
|
||||||
|
host: $1,
|
||||||
|
owner: $2,
|
||||||
|
name: $3
|
||||||
|
}
|
||||||
|
else
|
||||||
|
raise ArgumentError, "Invalid repository URL format: #{url}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Find or create repository from URL
|
||||||
|
def self.find_or_create_by_url(url)
|
||||||
|
parsed = parse_url(url)
|
||||||
|
find_or_create_by(url: url) do |repo|
|
||||||
|
repo.host = parsed[:host]
|
||||||
|
repo.owner = parsed[:owner]
|
||||||
|
repo.name = parsed[:name]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
64
app/services/repo_host/base_service.rb
Normal file
64
app/services/repo_host/base_service.rb
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
module RepoHost
|
||||||
|
class BaseService
|
||||||
|
def initialize(user, repo_url)
|
||||||
|
@user = user
|
||||||
|
@repo_url = repo_url
|
||||||
|
@owner, @repo = parse_repo_url(repo_url)
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetch_repo_metadata
|
||||||
|
raise NotImplementedError, "Subclasses must implement fetch_repo_metadata"
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
attr_reader :user, :repo_url, :owner, :repo
|
||||||
|
|
||||||
|
def parse_repo_url(url)
|
||||||
|
# Extract owner and repo from URL
|
||||||
|
# Example: https://github.com/owner/repo -> ["owner", "repo"]
|
||||||
|
# Example: https://gitlab.com/owner/repo -> ["owner", "repo"]
|
||||||
|
if url =~ %r{https?://[^/]+/([^/]+)/([^/]+)/?$}
|
||||||
|
[ $1, $2 ]
|
||||||
|
else
|
||||||
|
raise ArgumentError, "Invalid repository URL format: #{url}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def api_headers
|
||||||
|
raise NotImplementedError, "Subclasses must implement api_headers"
|
||||||
|
end
|
||||||
|
|
||||||
|
def make_api_request(url)
|
||||||
|
response = HTTP.headers(api_headers)
|
||||||
|
.timeout(connect: 5, read: 10)
|
||||||
|
.get(url)
|
||||||
|
|
||||||
|
handle_response(response)
|
||||||
|
end
|
||||||
|
|
||||||
|
def handle_response(response)
|
||||||
|
case response.status.code
|
||||||
|
when 200
|
||||||
|
response.parse
|
||||||
|
when 403
|
||||||
|
handle_rate_limit(response)
|
||||||
|
when 404
|
||||||
|
Rails.logger.warn "[#{self.class.name}] Repository #{owner}/#{repo} not found (404)"
|
||||||
|
nil
|
||||||
|
else
|
||||||
|
Rails.logger.error "[#{self.class.name}] API error. Status: #{response.status}"
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def handle_rate_limit(response)
|
||||||
|
if response.headers["X-RateLimit-Remaining"]&.to_i == 0
|
||||||
|
reset_time = Time.at(response.headers["X-RateLimit-Reset"].to_i)
|
||||||
|
delay_seconds = [ (reset_time - Time.current).ceil, 5 ].max
|
||||||
|
Rails.logger.warn "[#{self.class.name}] Rate limit exceeded. Reset in #{delay_seconds}s"
|
||||||
|
end
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
94
app/services/repo_host/github_service.rb
Normal file
94
app/services/repo_host/github_service.rb
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
require "http"
|
||||||
|
|
||||||
|
module RepoHost
|
||||||
|
class GithubService < BaseService
|
||||||
|
def fetch_repo_metadata
|
||||||
|
return nil unless user.github_access_token.present?
|
||||||
|
|
||||||
|
# Fetch basic repository info
|
||||||
|
repo_data = fetch_repository_info
|
||||||
|
return nil unless repo_data
|
||||||
|
|
||||||
|
# Fetch additional metadata
|
||||||
|
languages_data = fetch_languages
|
||||||
|
commits_data = fetch_recent_commits
|
||||||
|
commit_count = fetch_commit_count(repo_data["default_branch"])
|
||||||
|
|
||||||
|
{
|
||||||
|
stars: repo_data["stargazers_count"],
|
||||||
|
description: repo_data["description"],
|
||||||
|
language: repo_data["language"],
|
||||||
|
languages: languages_data&.keys&.join(", "),
|
||||||
|
homepage: repo_data["homepage"].presence,
|
||||||
|
commit_count: commit_count,
|
||||||
|
last_commit_at: commits_data&.first&.dig("commit", "committer", "date")&.then { |date| Time.parse(date) },
|
||||||
|
last_synced_at: Time.current
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def api_headers
|
||||||
|
{
|
||||||
|
"Accept" => "application/vnd.github.v3+json",
|
||||||
|
"Authorization" => "Bearer #{user.github_access_token}",
|
||||||
|
"X-GitHub-Api-Version" => "2022-11-28"
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetch_repository_info
|
||||||
|
url = "https://api.github.com/repos/#{owner}/#{repo}"
|
||||||
|
make_api_request(url)
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetch_languages
|
||||||
|
url = "https://api.github.com/repos/#{owner}/#{repo}/languages"
|
||||||
|
make_api_request(url)
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetch_recent_commits
|
||||||
|
# Get just the last few commits for metadata
|
||||||
|
url = "https://api.github.com/repos/#{owner}/#{repo}/commits?per_page=5"
|
||||||
|
make_api_request(url)
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetch_commit_count(default_branch = nil)
|
||||||
|
# GitHub API doesn't provide commit count directly, so we need to use a workaround
|
||||||
|
# We'll get the commit count from the commits endpoint with minimal data
|
||||||
|
branch_param = default_branch ? "&sha=#{default_branch}" : ""
|
||||||
|
url = "https://api.github.com/repos/#{owner}/#{repo}/commits?per_page=1#{branch_param}"
|
||||||
|
|
||||||
|
response = HTTP.headers(api_headers)
|
||||||
|
.timeout(connect: 5, read: 10)
|
||||||
|
.get(url)
|
||||||
|
|
||||||
|
case response.status.code
|
||||||
|
when 200
|
||||||
|
# Extract commit count from Link header pagination
|
||||||
|
link_header = response.headers["Link"]
|
||||||
|
if link_header
|
||||||
|
# Look for the "last" page number in the Link header
|
||||||
|
# Format: <https://api.github.com/repos/owner/repo/commits?page=962&per_page=1>; rel="last"
|
||||||
|
if match = link_header.match(/.*page=(\d+)[^>]*>;\s*rel="last"/)
|
||||||
|
match[1].to_i
|
||||||
|
else
|
||||||
|
# If no "last" link, there's only one page of commits
|
||||||
|
1
|
||||||
|
end
|
||||||
|
else
|
||||||
|
# No Link header means there's only one page
|
||||||
|
1
|
||||||
|
end
|
||||||
|
when 404
|
||||||
|
Rails.logger.warn "[#{self.class.name}] Repository #{owner}/#{repo} not found for commit count"
|
||||||
|
0
|
||||||
|
else
|
||||||
|
Rails.logger.warn "[#{self.class.name}] Failed to fetch commit count for #{owner}/#{repo}: #{response.status}"
|
||||||
|
0
|
||||||
|
end
|
||||||
|
rescue => e
|
||||||
|
Rails.logger.error "[#{self.class.name}] Error fetching commit count for #{owner}/#{repo}: #{e.message}"
|
||||||
|
0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
23
app/services/repo_host/service_factory.rb
Normal file
23
app/services/repo_host/service_factory.rb
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
module RepoHost
|
||||||
|
class ServiceFactory
|
||||||
|
def self.for_url(user, repo_url)
|
||||||
|
case repo_url
|
||||||
|
when %r{github\.com}
|
||||||
|
GithubService.new(user, repo_url)
|
||||||
|
else
|
||||||
|
raise ArgumentError, "Unsupported repository host: #{repo_url}. Currently only GitHub is supported."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.supported_hosts
|
||||||
|
%w[github.com]
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.host_for_url(repo_url)
|
||||||
|
uri = URI.parse(repo_url)
|
||||||
|
uri.host
|
||||||
|
rescue URI::InvalidURIError
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -6,17 +6,41 @@
|
|||||||
<% project_durations.each do |project| %>
|
<% project_durations.each do |project| %>
|
||||||
<div class="project-duration-card">
|
<div class="project-duration-card">
|
||||||
<div class="project-header">
|
<div class="project-header">
|
||||||
<strong title="<%= project[:project] %>">
|
<div class="project-name-section">
|
||||||
<%= (project[:project].presence || "Unknown").truncate(12) %>
|
<strong title="<%= project[:project] %>">
|
||||||
</strong>
|
<%= (project[:project].presence || "Unknown").truncate(15) %>
|
||||||
<% if project[:repo_url].present? %>
|
</strong>
|
||||||
<%= link_to "🔗", project[:repo_url], target: "_blank" %>
|
<% if project[:repository]&.stars.present? %>
|
||||||
<% end %>
|
<span class="project-stars">⭐ <%= project[:repository].stars %></span>
|
||||||
<% if current_user.github_uid.present? && project[:project].present? %>
|
<% end %>
|
||||||
<%= link_to "✏️", edit_my_project_repo_mapping_path(project_name: project[:project]), class: "edit-repo-link", data: { turbo_frame: '_top'} %>
|
</div>
|
||||||
<% end %>
|
<div class="project-actions">
|
||||||
|
<% if project[:repository]&.homepage.present? %>
|
||||||
|
<%= link_to "🌐", project[:repository].homepage, target: "_blank", title: "View project website" %>
|
||||||
|
<% end %>
|
||||||
|
<% if project[:repo_url].present? %>
|
||||||
|
<%= link_to "🔗", project[:repo_url], target: "_blank", title: "View repository" %>
|
||||||
|
<% end %>
|
||||||
|
<% if current_user.github_uid.present? && project[:project].present? %>
|
||||||
|
<%= link_to "✏️", edit_my_project_repo_mapping_path(project_name: project[:project]), class: "edit-repo-link", data: { turbo_frame: '_top'}, title: "Edit mapping" %>
|
||||||
|
<% end %>
|
||||||
|
</div>
|
||||||
<span class="project-time"><%= short_time_detailed project[:duration] %></span>
|
<span class="project-time"><%= short_time_detailed project[:duration] %></span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<% if project[:repository]&.description.present? %>
|
||||||
|
<div class="project-description">
|
||||||
|
<%= project[:repository].description.truncate(80) %>
|
||||||
|
</div>
|
||||||
|
<% end %>
|
||||||
|
|
||||||
|
<% if project[:repository]&.formatted_languages.present? %>
|
||||||
|
<div class="project-languages">
|
||||||
|
<span class="languages-label">Languages:</span>
|
||||||
|
<%= project[:repository].formatted_languages %>
|
||||||
|
</div>
|
||||||
|
<% end %>
|
||||||
|
|
||||||
<div class="project-progress-bar">
|
<div class="project-progress-bar">
|
||||||
<div
|
<div
|
||||||
class="progress"
|
class="progress"
|
||||||
@@ -24,6 +48,12 @@
|
|||||||
background-color: var(--primary-color);"
|
background-color: var(--primary-color);"
|
||||||
></div>
|
></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<% if project[:repository]&.last_commit_at.present? %>
|
||||||
|
<div class="project-meta">
|
||||||
|
Last commit: <%= time_ago_in_words(project[:repository].last_commit_at) %> ago
|
||||||
|
</div>
|
||||||
|
<% end %>
|
||||||
</div>
|
</div>
|
||||||
<% end %>
|
<% end %>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -141,6 +141,11 @@ Rails.application.configure do
|
|||||||
cleanup_successful_jobs: {
|
cleanup_successful_jobs: {
|
||||||
cron: "0 0 * * *",
|
cron: "0 0 * * *",
|
||||||
class: "CleanupSuccessfulJobsJob"
|
class: "CleanupSuccessfulJobsJob"
|
||||||
|
},
|
||||||
|
sync_stale_repo_metadata: {
|
||||||
|
cron: "0 4 * * *", # Daily at 4 AM
|
||||||
|
class: "SyncStaleRepoMetadataJob",
|
||||||
|
description: "Refreshes repository metadata (stars, commit counts, etc.) for repositories with stale data."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|||||||
20
db/migrate/20250530015016_create_repositories.rb
Normal file
20
db/migrate/20250530015016_create_repositories.rb
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
class CreateRepositories < ActiveRecord::Migration[8.0]
|
||||||
|
def change
|
||||||
|
create_table :repositories do |t|
|
||||||
|
t.string :url
|
||||||
|
t.string :host
|
||||||
|
t.string :owner
|
||||||
|
t.string :name
|
||||||
|
t.integer :stars
|
||||||
|
t.text :description
|
||||||
|
t.string :language
|
||||||
|
t.text :languages
|
||||||
|
t.integer :commit_count
|
||||||
|
t.datetime :last_commit_at
|
||||||
|
t.datetime :last_synced_at
|
||||||
|
|
||||||
|
t.timestamps
|
||||||
|
end
|
||||||
|
add_index :repositories, :url, unique: true
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
class UpdateProjectRepoMappingsToUseRepository < ActiveRecord::Migration[8.0]
|
||||||
|
def change
|
||||||
|
# Add repository reference
|
||||||
|
add_reference :project_repo_mappings, :repository, null: true, foreign_key: true
|
||||||
|
end
|
||||||
|
end
|
||||||
5
db/migrate/20250530015530_add_repository_to_commits.rb
Normal file
5
db/migrate/20250530015530_add_repository_to_commits.rb
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
class AddRepositoryToCommits < ActiveRecord::Migration[8.0]
|
||||||
|
def change
|
||||||
|
add_reference :commits, :repository, null: true, foreign_key: true
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
class AddHomepageToRepositories < ActiveRecord::Migration[8.0]
|
||||||
|
def change
|
||||||
|
add_column :repositories, :homepage, :string
|
||||||
|
end
|
||||||
|
end
|
||||||
29
db/schema.rb
generated
29
db/schema.rb
generated
@@ -10,9 +10,12 @@
|
|||||||
#
|
#
|
||||||
# It's strongly recommended that you check this file into your version control system.
|
# It's strongly recommended that you check this file into your version control system.
|
||||||
|
|
||||||
ActiveRecord::Schema[8.0].define(version: 2025_05_27_052632) do
|
ActiveRecord::Schema[8.0].define(version: 2025_05_30_135145) do
|
||||||
|
create_schema "pganalyze"
|
||||||
|
|
||||||
# These are extensions that must be enabled in order to support this database
|
# These are extensions that must be enabled in order to support this database
|
||||||
enable_extension "pg_catalog.plpgsql"
|
enable_extension "pg_catalog.plpgsql"
|
||||||
|
enable_extension "pg_stat_statements"
|
||||||
|
|
||||||
create_table "ahoy_events", force: :cascade do |t|
|
create_table "ahoy_events", force: :cascade do |t|
|
||||||
t.bigint "visit_id"
|
t.bigint "visit_id"
|
||||||
@@ -74,6 +77,8 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_27_052632) do
|
|||||||
t.jsonb "github_raw"
|
t.jsonb "github_raw"
|
||||||
t.datetime "created_at", null: false
|
t.datetime "created_at", null: false
|
||||||
t.datetime "updated_at", null: false
|
t.datetime "updated_at", null: false
|
||||||
|
t.bigint "repository_id"
|
||||||
|
t.index ["repository_id"], name: "index_commits_on_repository_id"
|
||||||
t.index ["user_id"], name: "index_commits_on_user_id"
|
t.index ["user_id"], name: "index_commits_on_user_id"
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -310,7 +315,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_27_052632) do
|
|||||||
t.string "repo_url", null: false
|
t.string "repo_url", null: false
|
||||||
t.datetime "created_at", null: false
|
t.datetime "created_at", null: false
|
||||||
t.datetime "updated_at", null: false
|
t.datetime "updated_at", null: false
|
||||||
|
t.bigint "repository_id"
|
||||||
t.index ["project_name"], name: "index_project_repo_mappings_on_project_name"
|
t.index ["project_name"], name: "index_project_repo_mappings_on_project_name"
|
||||||
|
t.index ["repository_id"], name: "index_project_repo_mappings_on_repository_id"
|
||||||
t.index ["user_id", "project_name"], name: "index_project_repo_mappings_on_user_id_and_project_name", unique: true
|
t.index ["user_id", "project_name"], name: "index_project_repo_mappings_on_user_id_and_project_name", unique: true
|
||||||
t.index ["user_id"], name: "index_project_repo_mappings_on_user_id"
|
t.index ["user_id"], name: "index_project_repo_mappings_on_user_id"
|
||||||
end
|
end
|
||||||
@@ -333,6 +340,24 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_27_052632) do
|
|||||||
t.index ["user_id"], name: "index_repo_host_events_on_user_id"
|
t.index ["user_id"], name: "index_repo_host_events_on_user_id"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
create_table "repositories", force: :cascade do |t|
|
||||||
|
t.string "url"
|
||||||
|
t.string "host"
|
||||||
|
t.string "owner"
|
||||||
|
t.string "name"
|
||||||
|
t.integer "stars"
|
||||||
|
t.text "description"
|
||||||
|
t.string "language"
|
||||||
|
t.text "languages"
|
||||||
|
t.integer "commit_count"
|
||||||
|
t.datetime "last_commit_at"
|
||||||
|
t.datetime "last_synced_at"
|
||||||
|
t.datetime "created_at", null: false
|
||||||
|
t.datetime "updated_at", null: false
|
||||||
|
t.string "homepage"
|
||||||
|
t.index ["url"], name: "index_repositories_on_url", unique: true
|
||||||
|
end
|
||||||
|
|
||||||
create_table "sailors_log_leaderboards", force: :cascade do |t|
|
create_table "sailors_log_leaderboards", force: :cascade do |t|
|
||||||
t.string "slack_channel_id"
|
t.string "slack_channel_id"
|
||||||
t.string "slack_uid"
|
t.string "slack_uid"
|
||||||
@@ -428,6 +453,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_27_052632) do
|
|||||||
end
|
end
|
||||||
|
|
||||||
add_foreign_key "api_keys", "users"
|
add_foreign_key "api_keys", "users"
|
||||||
|
add_foreign_key "commits", "repositories"
|
||||||
add_foreign_key "commits", "users"
|
add_foreign_key "commits", "users"
|
||||||
add_foreign_key "email_addresses", "users"
|
add_foreign_key "email_addresses", "users"
|
||||||
add_foreign_key "email_verification_requests", "users"
|
add_foreign_key "email_verification_requests", "users"
|
||||||
@@ -437,6 +463,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_27_052632) do
|
|||||||
add_foreign_key "leaderboard_entries", "users"
|
add_foreign_key "leaderboard_entries", "users"
|
||||||
add_foreign_key "mailing_addresses", "users"
|
add_foreign_key "mailing_addresses", "users"
|
||||||
add_foreign_key "physical_mails", "users"
|
add_foreign_key "physical_mails", "users"
|
||||||
|
add_foreign_key "project_repo_mappings", "repositories"
|
||||||
add_foreign_key "project_repo_mappings", "users"
|
add_foreign_key "project_repo_mappings", "users"
|
||||||
add_foreign_key "repo_host_events", "users"
|
add_foreign_key "repo_host_events", "users"
|
||||||
add_foreign_key "sign_in_tokens", "users"
|
add_foreign_key "sign_in_tokens", "users"
|
||||||
|
|||||||
27
test/fixtures/repositories.yml
vendored
Normal file
27
test/fixtures/repositories.yml
vendored
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
|
||||||
|
|
||||||
|
one:
|
||||||
|
url: MyString
|
||||||
|
host: MyString
|
||||||
|
owner: MyString
|
||||||
|
name: MyString
|
||||||
|
stars: 1
|
||||||
|
description: MyText
|
||||||
|
language: MyString
|
||||||
|
languages: MyText
|
||||||
|
commit_count: 1
|
||||||
|
last_commit_at: 2025-05-30 01:50:16
|
||||||
|
last_synced_at: 2025-05-30 01:50:16
|
||||||
|
|
||||||
|
two:
|
||||||
|
url: MyString
|
||||||
|
host: MyString
|
||||||
|
owner: MyString
|
||||||
|
name: MyString
|
||||||
|
stars: 1
|
||||||
|
description: MyText
|
||||||
|
language: MyString
|
||||||
|
languages: MyText
|
||||||
|
commit_count: 1
|
||||||
|
last_commit_at: 2025-05-30 01:50:16
|
||||||
|
last_synced_at: 2025-05-30 01:50:16
|
||||||
7
test/models/repository_test.rb
Normal file
7
test/models/repository_test.rb
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
require "test_helper"
|
||||||
|
|
||||||
|
class RepositoryTest < ActiveSupport::TestCase
|
||||||
|
# test "the truth" do
|
||||||
|
# assert true
|
||||||
|
# end
|
||||||
|
end
|
||||||
Reference in New Issue
Block a user