|
complete rewrite
|
1 |
#!/bin/bash |
| 2 | ||
| 3 |
hangar[path]=$HOME/.fgfs/flightgear-fgaddon/Aircraft |
|
| 4 | ||
| 5 |
hangar[name]=FGMEMBERS |
|
| 6 |
hangar[source]=github |
|
| 7 |
hangar[type]=git |
|
|
fix fgmember url
|
8 |
hangar[url]=https://github.com/FGMEMBERS |
|
complete rewrite
|
9 |
hangar[active]=1 |
| 10 | ||
| 11 |
function getfromrepo () {
|
|
| 12 |
curl -s https://raw.githubusercontent.com/FGMEMBERS/${1%%/*}/master/${1#*/}
|
|
| 13 |
} |
|
| 14 | ||
| 15 |
function on_exit () {
|
|
| 16 |
rm -f $gh_curl_content $json_file $contents |
|
| 17 |
unset contents gh_curl_content |
|
| 18 |
} |
|
| 19 | ||
| 20 |
function parse_repo_history () {
|
|
| 21 |
function github_curl () {
|
|
| 22 |
test $# -eq 0 && return $(test -n "$githup_token") |
|
| 23 |
curl ${githup_token:+-u $githup_token} -si $1 > $gh_curl_content
|
|
|
header specification are cas...
|
24 |
eval $(sed -rn '1,/^\s*$/{s/^X-Ratelimit-Remaining:\s*([0-9]+).*$/remaining=\1/ip;s/^X-Ratelimit-Reset:\s*([0-9]+).*$/reset=\1/ip}' $gh_curl_content)
|
|
complete rewrite
|
25 |
if test ${remaining:-1} -eq 0; then
|
| 26 |
((reset = reset + 10)) # just to be prudent |
|
| 27 |
echo "github API limit: waiting $(date +%H:%M -d@$reset) to continue" >&2 |
|
| 28 |
if [[ $1 =~ '/contents/'$ ]]; then |
|
| 29 |
echo "process already found data so far" >&2 |
|
| 30 |
apply_revision >&2 |
|
| 31 |
fi |
|
| 32 |
sleep $(( $reset - $(date +%s) )) |
|
| 33 |
curl ${githup_token:+-u $githup_token} -s $1 # not sure that was really fetched
|
|
| 34 |
else |
|
| 35 |
sed -n '/^\s*$/,$p' $gh_curl_content |
|
| 36 |
fi |
|
| 37 |
} |
|
| 38 | ||
| 39 |
gh_curl_content=$temppath/github-$tempid |
|
| 40 |
contents=$temppath/contents-$tempid |
|
| 41 |
github_orga_repos='https://api.github.com/orgs/FGMEMBERS/repos?sort=updated&type=all&per_page=100&page=_page_' |
|
| 42 |
page=1 |
|
| 43 | ||
| 44 |
echo '[' > ${json_file}_recent
|
|
| 45 | ||
| 46 |
while github_curl "${github_orga_repos/_page_/$((page++))}" > $json_file; do
|
|
| 47 |
jq_length=$(json 'length') |
|
| 48 |
test $jq_length -eq 0 && break |
|
| 49 | ||
| 50 |
for ((i = 0; i < $jq_length; i++)); do |
|
| 51 |
if test $(date +%s -d $(json ".[$i].updated_at")) -gt $latest_revision; then |
|
| 52 |
json ".[$i]" >> ${json_file}_recent
|
|
| 53 |
echo ',' >> ${json_file}_recent
|
|
| 54 |
else |
|
| 55 |
break 2 |
|
| 56 |
fi |
|
| 57 |
done |
|
| 58 |
done |
|
| 59 | ||
| 60 |
sed -i '${/^,/d}' ${json_file}_recent
|
|
| 61 |
echo ']' >> ${json_file}_recent
|
|
| 62 | ||
| 63 |
mv -f ${json_file}_recent $json_file
|
|
| 64 |
jq_length=$(json 'length') |
|
| 65 |
local progress=0 |
|
| 66 |
local repo |
|
| 67 | ||
| 68 |
if test $latest_revision -eq 1; then |
|
| 69 |
if github_curl; then |
|
| 70 |
max_requests_per_hour=5000 |
|
| 71 |
else |
|
| 72 |
max_requests_per_hour=60 |
|
| 73 |
fi |
|
| 74 |
echo "the initial import may take more than $(($jq_length / $max_requests_per_hour)) hours to perform" |
|
| 75 |
fi |
|
| 76 | ||
| 77 |
for ((i = 0; i < $jq_length; i++)); do |
|
| 78 |
local repo=$(json ".[$i].name") |
|
| 79 |
add_record name $repo |
|
| 80 | ||
| 81 |
github_curl "https://api.github.com/repos/FGMEMBERS/${repo}/contents/" > $contents
|
|
| 82 |
for sx in $(json '.[] | select(.type == "file") | .path | capture("(?<setxml>.+)-set.xml") | .setxml' $contents); do
|
|
| 83 |
add_setxml_for_aircraft $repo $sx |
|
| 84 |
done |
|
| 85 | ||
| 86 |
if test -n "$sx"; then |
|
| 87 |
add_record revision $(date +%s -d $(json ".[$i].updated_at")) |
|
| 88 |
add_record date $(date +%s -d $(json ".[$i].updated_at")) |
|
|
remove hard coded value
|
89 |
add_record author ${hangar[name]}
|
|
complete rewrite
|
90 |
add_aircraft |
| 91 |
else |
|
| 92 |
sqlite_request "delete from aircrafts where name = '$repo' and hangar = ${hangar[i]}"
|
|
| 93 |
fi |
|
| 94 | ||
| 95 |
newprogress=$((i * 100 / $jq_length)) |
|
| 96 |
if test $(( $newprogress - $progress )) -ge ${progress_granularity:-1}; then
|
|
| 97 |
progress=$newprogress |
|
| 98 |
echo "$progress% ($(sqlite_request 'select count(name) from recover_aircrafts'))" |
|
| 99 |
fi |
|
| 100 |
done |
|
| 101 | ||
| 102 |
apply_revision |
|
| 103 |
} |