| 1 | #!/usr/bin/env bash |
| 2 | # |
| 3 | # sync-repos - Sync all GitHub organizations and their repos locally |
| 4 | # |
| 5 | # Clones new repos and pulls updates (ff-only) for existing ones. |
| 6 | # Config: ~/.config/sync-repos/orgs.conf |
| 7 | # |
| 8 | |
| 9 | set -uo pipefail |
| 10 | |
| 11 | CONFIG_FILE="${SYNC_REPOS_CONFIG:-$HOME/.config/sync-repos/orgs.conf}" |
| 12 | GITHUB_ORGS_DIR="${GITHUB_ORGS_DIR:-$HOME/GithubOrgs}" |
| 13 | CLONE_TIMEOUT="${CLONE_TIMEOUT:-300}" # seconds (5 min for large repos) |
| 14 | |
| 15 | # Colors |
| 16 | RED='\033[0;31m' |
| 17 | GREEN='\033[0;32m' |
| 18 | YELLOW='\033[0;33m' |
| 19 | BLUE='\033[0;34m' |
| 20 | BOLD='\033[1m' |
| 21 | NC='\033[0m' |
| 22 | |
| 23 | # Counters |
| 24 | total_cloned=0 |
| 25 | total_updated=0 |
| 26 | total_skipped=0 |
| 27 | total_failed=0 |
| 28 | declare -a failed_repos=() |
| 29 | |
| 30 | log_info() { echo -e "${BLUE}→${NC} $*"; } |
| 31 | log_success() { echo -e "${GREEN}✓${NC} $*"; } |
| 32 | log_warn() { echo -e "${YELLOW}⚠${NC} $*"; } |
| 33 | log_error() { echo -e "${RED}✗${NC} $*"; } |
| 34 | log_header() { echo -e "\n${BOLD}${BLUE}══════════════════════════════════════${NC}"; echo -e "${BOLD} $*${NC}"; echo -e "${BOLD}${BLUE}══════════════════════════════════════${NC}"; } |
| 35 | |
| 36 | check_requirements() { |
| 37 | if ! command -v gh &> /dev/null; then |
| 38 | log_error "gh CLI is not installed. Install from: https://cli.github.com/" |
| 39 | exit 1 |
| 40 | fi |
| 41 | |
| 42 | if ! gh auth status &> /dev/null; then |
| 43 | log_error "Not authenticated with GitHub. Run: gh auth login" |
| 44 | exit 1 |
| 45 | fi |
| 46 | |
| 47 | if [[ ! -f "$CONFIG_FILE" ]]; then |
| 48 | log_error "Config file not found: $CONFIG_FILE" |
| 49 | exit 1 |
| 50 | fi |
| 51 | } |
| 52 | |
| 53 | sync_submodules() { |
| 54 | local repo_path="$1" |
| 55 | local repo="$2" |
| 56 | |
| 57 | [[ -f "$repo_path/.gitmodules" ]] || return 0 |
| 58 | |
| 59 | ( |
| 60 | cd "$repo_path" 2>/dev/null || exit 0 |
| 61 | # Prefix `-` = uninitialized, `+` = checked-out commit differs from index. |
| 62 | # Either means `submodule update --init --recursive` has work to do. |
| 63 | if git submodule status --recursive 2>/dev/null | grep -qE '^[-+]'; then |
| 64 | log_info " $repo: syncing submodules..." |
| 65 | if timeout "$CLONE_TIMEOUT" git submodule update --init --recursive 2>&1 >/dev/null; then |
| 66 | log_success " $repo: submodules initialized" |
| 67 | else |
| 68 | log_warn " $repo: submodule update failed" |
| 69 | fi |
| 70 | fi |
| 71 | ) |
| 72 | } |
| 73 | |
| 74 | sync_repo() { |
| 75 | local github_org="$1" |
| 76 | local repo="$2" |
| 77 | local org_path="$3" |
| 78 | local repo_path="${org_path}/${repo}" |
| 79 | |
| 80 | # Case 1: Valid git repo exists - try to update |
| 81 | if [[ -d "$repo_path/.git" ]]; then |
| 82 | # Use subshell to avoid cd pollution, capture result |
| 83 | local result |
| 84 | result=$( |
| 85 | cd "$repo_path" 2>/dev/null || { echo "invalid"; exit 0; } |
| 86 | |
| 87 | # Verify it's a valid git repo with a HEAD |
| 88 | if ! git rev-parse HEAD &>/dev/null; then |
| 89 | echo "invalid" |
| 90 | exit 0 |
| 91 | fi |
| 92 | |
| 93 | # Check for uncommitted changes (staged or unstaged) |
| 94 | if ! git diff --quiet 2>/dev/null || ! git diff --cached --quiet 2>/dev/null; then |
| 95 | echo "dirty" |
| 96 | exit 0 |
| 97 | fi |
| 98 | |
| 99 | # Try fast-forward pull |
| 100 | if pull_output=$(timeout 30 git pull --ff-only 2>&1); then |
| 101 | if [[ "$pull_output" == *"Already up to date"* ]]; then |
| 102 | echo "uptodate" |
| 103 | else |
| 104 | echo "updated" |
| 105 | fi |
| 106 | else |
| 107 | echo "diverged" |
| 108 | fi |
| 109 | ) |
| 110 | |
| 111 | case "$result" in |
| 112 | invalid) |
| 113 | log_warn "$repo (invalid git repo, skipping)" |
| 114 | ((total_skipped++)) || true |
| 115 | ;; |
| 116 | dirty) |
| 117 | log_warn "$repo (uncommitted changes, skipping)" |
| 118 | ((total_skipped++)) || true |
| 119 | ;; |
| 120 | uptodate) |
| 121 | log_success "$repo (up to date)" |
| 122 | sync_submodules "$repo_path" "$repo" |
| 123 | ;; |
| 124 | updated) |
| 125 | log_success "$repo (updated)" |
| 126 | ((total_updated++)) || true |
| 127 | sync_submodules "$repo_path" "$repo" |
| 128 | ;; |
| 129 | diverged) |
| 130 | log_warn "$repo (can't fast-forward, skipping)" |
| 131 | ((total_skipped++)) || true |
| 132 | ;; |
| 133 | *) |
| 134 | log_warn "$repo (unknown state: $result)" |
| 135 | ((total_skipped++)) || true |
| 136 | ;; |
| 137 | esac |
| 138 | return 0 |
| 139 | |
| 140 | # Case 2: Directory exists but no .git |
| 141 | elif [[ -d "$repo_path" ]]; then |
| 142 | if [[ -z "$(ls -A "$repo_path" 2>/dev/null)" ]]; then |
| 143 | log_info "$repo (empty dir, removing and cloning...)" |
| 144 | rmdir "$repo_path" |
| 145 | else |
| 146 | log_warn "$repo (exists but not a git repo)" |
| 147 | ((total_skipped++)) || true |
| 148 | return 0 |
| 149 | fi |
| 150 | fi |
| 151 | |
| 152 | # Case 3: Clone new repo with retry logic |
| 153 | local ssh_url="git@github.com:${github_org}/${repo}.git" |
| 154 | local max_retries=3 |
| 155 | local attempt=1 |
| 156 | local clone_success=false |
| 157 | |
| 158 | while [[ $attempt -le $max_retries ]] && [[ "$clone_success" == "false" ]]; do |
| 159 | # Clean up any partial clone |
| 160 | [[ -d "$repo_path" ]] && rm -rf "$repo_path" |
| 161 | |
| 162 | if [[ $attempt -eq 1 ]]; then |
| 163 | log_info "Cloning $repo (SSH)..." |
| 164 | else |
| 165 | log_info "Cloning $repo (retry $attempt/$max_retries)..." |
| 166 | fi |
| 167 | |
| 168 | # Try full clone first, then shallow on final attempt |
| 169 | local clone_args="--progress" |
| 170 | if [[ $attempt -eq $max_retries ]]; then |
| 171 | clone_args="--progress --depth 1" |
| 172 | log_info "(trying shallow clone...)" |
| 173 | fi |
| 174 | |
| 175 | if timeout "$CLONE_TIMEOUT" git clone $clone_args "$ssh_url" "$repo_path" 2>&1; then |
| 176 | if [[ -d "$repo_path/.git" ]]; then |
| 177 | if [[ $attempt -eq $max_retries ]]; then |
| 178 | log_success "$repo (cloned shallow)" |
| 179 | else |
| 180 | log_success "$repo (cloned)" |
| 181 | fi |
| 182 | ((total_cloned++)) || true |
| 183 | clone_success=true |
| 184 | sync_submodules "$repo_path" "$repo" |
| 185 | fi |
| 186 | else |
| 187 | local exit_code=$? |
| 188 | if [[ -d "$repo_path/.git" ]]; then |
| 189 | log_success "$repo (cloned)" |
| 190 | ((total_cloned++)) || true |
| 191 | clone_success=true |
| 192 | sync_submodules "$repo_path" "$repo" |
| 193 | elif [[ $exit_code -eq 124 ]]; then |
| 194 | log_warn "$repo (attempt $attempt timed out)" |
| 195 | else |
| 196 | log_warn "$repo (attempt $attempt failed)" |
| 197 | fi |
| 198 | fi |
| 199 | |
| 200 | ((attempt++)) || true |
| 201 | done |
| 202 | |
| 203 | if [[ "$clone_success" == "false" ]]; then |
| 204 | log_error "$repo (clone failed after $max_retries attempts)" |
| 205 | failed_repos+=("${github_org}/${repo}") |
| 206 | ((total_failed++)) || true |
| 207 | fi |
| 208 | } |
| 209 | |
| 210 | sync_org() { |
| 211 | local local_dir="$1" |
| 212 | local org_spec="$2" |
| 213 | local org_path="${GITHUB_ORGS_DIR}/${local_dir}" |
| 214 | |
| 215 | # Parse org:repo1,repo2 format |
| 216 | local github_org specific_repos |
| 217 | if [[ "$org_spec" == *:* ]]; then |
| 218 | github_org="${org_spec%%:*}" |
| 219 | specific_repos="${org_spec#*:}" |
| 220 | else |
| 221 | github_org="$org_spec" |
| 222 | specific_repos="" |
| 223 | fi |
| 224 | |
| 225 | log_header "$local_dir ← $github_org" |
| 226 | |
| 227 | # Create org directory if needed |
| 228 | if [[ ! -d "$org_path" ]]; then |
| 229 | log_info "Creating directory: $org_path" |
| 230 | mkdir -p "$org_path" |
| 231 | fi |
| 232 | |
| 233 | local repos |
| 234 | if [[ -n "$specific_repos" ]]; then |
| 235 | # Use specific repos from config |
| 236 | repos=$(echo "$specific_repos" | tr ',' '\n') |
| 237 | local repo_count |
| 238 | repo_count=$(echo "$repos" | wc -l) |
| 239 | log_info "Syncing $repo_count specific repositories" |
| 240 | else |
| 241 | # Fetch all repos from GitHub |
| 242 | log_info "Fetching repository list..." |
| 243 | if ! repos=$(gh repo list "$github_org" --limit 1000 --json name --jq '.[].name' 2>&1); then |
| 244 | log_error "Failed to list repos for $github_org: $repos" |
| 245 | return 1 |
| 246 | fi |
| 247 | |
| 248 | if [[ -z "$repos" ]]; then |
| 249 | log_warn "No repositories found for $github_org" |
| 250 | return 0 |
| 251 | fi |
| 252 | |
| 253 | local repo_count |
| 254 | repo_count=$(echo "$repos" | wc -l) |
| 255 | log_info "Found $repo_count repositories" |
| 256 | fi |
| 257 | echo "" |
| 258 | |
| 259 | local cloned=0 updated=0 skipped=0 failed=0 |
| 260 | local before_cloned=$total_cloned |
| 261 | local before_updated=$total_updated |
| 262 | local before_skipped=$total_skipped |
| 263 | local before_failed=$total_failed |
| 264 | |
| 265 | while IFS= read -r repo; do |
| 266 | [[ -z "$repo" ]] && continue |
| 267 | sync_repo "$github_org" "$repo" "$org_path" |
| 268 | done <<< "$repos" |
| 269 | |
| 270 | cloned=$((total_cloned - before_cloned)) |
| 271 | updated=$((total_updated - before_updated)) |
| 272 | skipped=$((total_skipped - before_skipped)) |
| 273 | failed=$((total_failed - before_failed)) |
| 274 | |
| 275 | echo "" |
| 276 | echo " Cloned: $cloned | Updated: $updated | Skipped: $skipped | Failed: $failed" |
| 277 | } |
| 278 | |
| 279 | main() { |
| 280 | echo -e "${BOLD}${BLUE}" |
| 281 | echo " ╔═══════════════════════════════════╗" |
| 282 | echo " ║ GitHub Repos Sync ║" |
| 283 | echo " ╚═══════════════════════════════════╝" |
| 284 | echo -e "${NC}" |
| 285 | |
| 286 | check_requirements |
| 287 | |
| 288 | log_info "Config: $CONFIG_FILE" |
| 289 | log_info "Target: $GITHUB_ORGS_DIR" |
| 290 | |
| 291 | # Create base directory if needed |
| 292 | mkdir -p "$GITHUB_ORGS_DIR" |
| 293 | |
| 294 | # Read config and sync each org |
| 295 | while IFS='=' read -r local_dir org_spec || [[ -n "$local_dir" ]]; do |
| 296 | # Skip comments and empty lines |
| 297 | [[ -z "$local_dir" || "$local_dir" =~ ^[[:space:]]*# ]] && continue |
| 298 | |
| 299 | # Trim whitespace |
| 300 | local_dir="${local_dir// /}" |
| 301 | org_spec="${org_spec// /}" |
| 302 | |
| 303 | sync_org "$local_dir" "$org_spec" |
| 304 | done < "$CONFIG_FILE" |
| 305 | |
| 306 | # Summary |
| 307 | log_header "Sync Complete" |
| 308 | echo "" |
| 309 | echo -e " ${GREEN}Cloned:${NC} $total_cloned" |
| 310 | echo -e " ${BLUE}Updated:${NC} $total_updated" |
| 311 | echo -e " ${YELLOW}Skipped:${NC} $total_skipped" |
| 312 | echo -e " ${RED}Failed:${NC} $total_failed" |
| 313 | echo "" |
| 314 | |
| 315 | if [[ ${#failed_repos[@]} -gt 0 ]]; then |
| 316 | log_error "Failed repositories:" |
| 317 | for repo in "${failed_repos[@]}"; do |
| 318 | echo " - $repo" |
| 319 | done |
| 320 | fi |
| 321 | } |
| 322 | |
| 323 | main "$@" |