1 contributor
#!/bin/bash
set -e
declare -A data=(
[/sim/description]=text
[/sim/long-description]=text
[/sim/author]=text
[/sim/flight-model]=text
[/sim/type]=text
[/sim/model/path]=text
)
missing_data_check=( /sim/model/path )
database=${DB:-$0.db}
test -r "$0.conf" && source "$0.conf"
#locale=fr
tempid=$(mktemp --dry-run XXXXXXX)
temppath=/dev/shm
setxml=$temppath/setxml-$tempid
json_file=$temppath/github_json-$tempid
in_ram_database=$temppath/${database##*/}-$tempid
function xmlgetnext () {
local IFS='>'
read -d '<' TAG VALUE
# by design, the first TAG/VALUE pair is empty
# to avoid infinite loops at end of file parsing we return an error
# the next time we find an empty TAG
if test -z "$TAG"; then
test ${xmlgetnext_empty_tag:-0} -gt 0 && return 1
xmlgetnext_empty_tag=$(( xmlgetnext_empty_tag + 1 ))
fi
# process $TAG only if necessary
local _TAG=$(printf '%q' $TAG)
if test ${_TAG:0:1} = '$'; then
TAG=$(tr '\n' ' ' <<< $TAG | sed 's/ */ /g; s/ *$//')
fi
}
function json () {
jq --raw-output "$1" < ${2:-${json_file:?}}
}
rm -f $temppath/sqlite_request
function sqlite_request () {
echo -e "## REQ $(( ++sqlite_request_count ))\n${1}\n" >> $temppath/sqlite_request
sqlite3 "$in_ram_database" <<< "$1"
}
function xmlremovecomments () {
sed -ri 's/<(!--|script>)/\n&/;s/(<\/script|--)>/&\n/' $setxml
sed -ri '/<(script>|!--).*(<\/script|--)>/d;/<(script>|!--)/,/(<\/script|--)>/d' $setxml
sed -i 's/\xef\xbb\xbf//;s/\r//' $setxml # removes BOM and ^M
}
function trap_break () {
trap '' INT
echo "stop requested"
}
function trap_exit () {
trapped_rc=$?
trap '' INT
if declare -f on_exit > /dev/null; then
on_exit
fi
if test ! -e $in_ram_database; then
exit
fi
echo "updating installation status"
for ac in $(sqlite_request 'select printf("%i:%s/%s", aircrafts.id, aircrafts.name, setxml.file)
from aircrafts inner join setxml
where aircrafts.id = setxml.variantof and setxml.installed != 0;'); do
ac_path=${ac#*:}
if test ! -e ${hangar[path]}/$ac_path-set.xml; then
sqlite_request "update setxml set installed = 0 where file = '${ac_path#*/}' and variantof = ${ac%:*}"
fi
done
for ac in ${hangar[path]}/*/*-set.xml; do
ac=${ac/${hangar[path]}}
sx=${ac##*/}
ac=${ac%/*}
if test -d ${hangar[path]}/$ac/.svn; then
install_type=1
elif test -d ${hangar[path]}/$ac/.git; then
install_type=2
else
install_type=3
fi
sqlite_request "update setxml set installed = $install_type
where exists (
select 1
from aircrafts
where name = '${ac/\/}' and setxml.variantof = id
)"
done
local missing_setxml=$(sqlite_request "select printf(' - %s (%s)', aircrafts.name, hangars.name)
from aircrafts inner join hangars
where hangars.id = aircrafts.hangar and aircrafts.id not in (select variantof from setxml)")
if test -n "$missing_setxml"; then
echo -e "missing setxml config for :\n$missing_setxml"
fi
for data_presence_check in ${missing_data_check[@]}; do
if [[ -v data[$data_presence_check] ]]; then
local missing_data=$(sqlite_request "select count(setxml.file)
from aircrafts inner join setxml
where aircrafts.id = setxml.variantof and setxml.\`$data_presence_check\` = ''")
if test $missing_data -gt 0; then
echo "$missing_data aircrafts without $data_presence_check information"
if test $missing_data -le 10; then
echo "aircrafts without $data_presence_check information:"
sqlite_request "select printf(' - %s/%s (%s)', aircrafts.name, setxml.file, hangars.name)
from aircrafts inner join setxml, hangars
where
aircrafts.id = setxml.variantof
and
aircrafts.hangar = hangars.id
and
setxml.\`$data_presence_check\` = ''"
fi
fi
fi
done
if test -r "$database" && md5sum $in_ram_database | sed "s,$in_ram_database,$database," | md5sum --status -c -; then
echo "no changes in $database"
elif test -w "$database"; then
rm -f "$database"
sqlite_request '.dump' | sqlite3 "$database"
echo "database $database updated"
elif test ! -e "$database" -a -w ${database%/*}; then
sqlite_request '.dump' | sqlite3 "$database"
echo "database $database created"
else
echo "nothing can be done with $database !"
fi
rm -f $temppath/*-$tempid
}
function update_database () {
sqlite_request "insert into aircrafts (name, author, revision, date, hangar)
select name, author, revision, date, hangar from recover_aircrafts
where recover_aircrafts.name = '$ac' and recover_aircrafts.hangar = ${hangar[id]}
on conflict (name, hangar) where aircrafts.name = '$ac' and aircrafts.hangar = ${hangar[id]} do
update set
author = (select author from recover_aircrafts where name = '$ac'),
revision = (select revision from recover_aircrafts where name = '$ac'),
date = (select date from recover_aircrafts where name = '$ac')
where aircrafts.name = '$ac' and aircrafts.hangar = ${hangar[id]}"
id=$(sqlite_request "select id from aircrafts where name is '${ac}' and hangar = ${hangar[id]}")
echo $(sqlite_request "select printf('[ %i/%i ] $ac', count(sx), count(distinct ac)) from recover_setxml")
for sx in $(sqlite_request "select distinct sx from recover_setxml where ac = '$ac'"); do
for col in ${!data[@]}; do
data[$col]=
done
echo " -> $sx"
getfromrepo ${ac}/$sx-set.xml > $setxml
unset xmlgetnext_empty_tag property include include_rootpath ac_save
while xmlgetnext; do
if [[ "$TAG" =~ ^"PropertyList include=" ]]; then
include_rootpath=${include%/*}
test $include = $include_rootpath && unset include_rootpath
eval $(echo ${TAG#* }) # include="..."
if [[ "$include" =~ ^Aircraft/Generic/ ]]; then
unset include include_rootpath
continue
elif [[ "$include" =~ ^'../' ]]; then
if test -n "$include_rootpath"; then
if [[ "$include_rootpath" =~ '/' ]]; then
include_rootpath=${include_rootpath%/*}
else
unset include_rootpath
fi
else
ac_save=$ac
unset ac
fi
include=${include/\.\.\/}
fi
getfromrepo ${ac}/${include_rootpath:+$include_rootpath/}$include >> $setxml
fi
done < $setxml
test -n "$ac_save" && ac=$ac_save
# some aircrafts (mostly from the helijah's files architecture template)
# break because of infinite loop in middle of file
# I can't find the reason of this infinite loop
# this is the reason of this double-pass
unset xmlgetnext_empty_tag property
while xmlgetnext; do
case "${TAG:0:1}" in
''|'?'|'!')
continue;;
/)
property=${property%/*};;
*)
if test "${TAG: -1}" != '/'; then
property+=/${TAG%% *}
fi;;
esac
if [[ "$property" = /PropertyList@($data_pattern) ]]; then
if test -z "${data[${property/\/PropertyList}]}"; then
eval "data[${property/\/PropertyList}]=\"${VALUE//\"/\\\"}\""
data[${property/\/PropertyList}]=$(tr '\n' ' ' <<< ${data[${property/\/PropertyList}]} | sed -r 's/^\s*//;s/\s+/ /g;s/\s*$//')
fi
fi
# continue parsing (while loop) until everything's found
for col in ${!data[@]}; do
test -z "${data[$col]}" && continue 2
done
break # everything's found
done < $setxml
if eval "test -z \"$data_test_null\""; then
echo "WARNING: no info found, skipping"
mkdir -p $temppath/no-data-ac
cp -f $setxml $temppath/no-data-ac/${ac}-${sx}
else
insert_values="'$sx', $id, "
insert_col='file, variantof, '
update_values=''
for col in ${!data[@]}; do
insert_col+="\`$col\`, "
insert_values+="'${data[$col]//\'/\'\'}', "
if test -n "${data[$col]}"; then
update_values+="\`$col\` = '${data[$col]//\'/\'\'}', "
fi
done
sqlite_request "insert into setxml (${insert_col%,*}, installed) values (${insert_values%,*}, 0)
on conflict (file, variantof) where file = '$sx' and variantof = $id do
update set
${update_values%,*}, installed = 0
where
file = '$sx' and variantof = $id"
fi
sqlite_request "delete from recover_setxml where ac = '$ac' and sx = '$sx'"
done
}
function add_record () {
ac_ass_array[$1]="$2"
}
function get_record () {
if test -n "$1"; then
echo "${ac_ass_array[$1]}"
else
for k in ${!ac_ass_array[@]}; do
echo $k = ${ac_ass_array[$k]}
done
fi
}
function add_aircraft () {
for key in name revision date author; do
test -n "${ac_ass_array[$key]}" # exit if missing data (with the help of "set -e")
done
local new_revision=$(sqlite_request "select revision from recover_aircrafts
where name = '${ac_ass_array[name]}'")
if test -z "${new_revision}"; then
sqlite_request "insert into recover_aircrafts (name, revision, date, author, hangar)
values (
'${ac_ass_array[name]}',
${ac_ass_array[revision]},
${ac_ass_array[date]},
'${ac_ass_array[author]}',
${hangar[id]})"
elif test ${new_revision} -lt ${ac_ass_array[revision]//\"}; then
sqlite_request "update recover_aircrafts
set
revision = ${ac_ass_array[revision]},
date = ${ac_ass_array[date]},
author = '${ac_ass_array[author]}',
hangar = ${hangar[id]}
where name = '${ac_ass_array[name]}'"
fi
for key in name revision date author; do
ac_ass_array[$key]=''
done
}
function add_setxml_for_aircraft () {
sqlite_request "insert into recover_setxml values ('$1', '${2/%-set.xml}')
on conflict (ac, sx) where ac = '$1' and sx = '${2/%-set.xml}'
do nothing"
}
function apply_revision () {
for ac in $(sqlite_request "select name from recover_aircrafts"); do
# delete aircrafts that have been deleted from the repo
sqlite_request "delete from setxml
where (file, variantof) in (
select file, variantof from setxml
inner join aircrafts
where aircrafts.id = setxml.variantof
and aircrafts.name = '$ac'
and aircrafts.hangar = ${hangar[id]}
and setxml.file not in (
select sx from recover_setxml where ac = '$ac'
)
)"
# delete aircrafts without setxml found
sqlite_request "delete from recover_aircrafts
where name not in (select distinct ac from recover_setxml)"
update_database
if test -d ${hangar[path]}/${ac}/.${hangar[type]} \
&& \
case ${hangar[type]} in
svn) test "$(svn info --show-item=url ${hangar[path]}/${ac})" != "${hangar[url]}/${ac}";;
git) test "$(git -C ${hangar[path]}/${ac} config --get remote.origin.url)" != "${hangar[url]}/${ac}.git";;
esac \
|| test -d ${hangar[path]}/${ac} -a ! -d ${hangar[path]}/${ac}/.${hangar[type]}; then
echo "INFO: local ${ac} installed out from repo" >&2
fi
sqlite_request "delete from recover_aircrafts where name = '$ac'"
done
}
trap trap_break INT
trap trap_exit EXIT
stty -echoctl
declare -A hangar
data_pattern=$(printf "%s|" ${!data[@]})
data_pattern=${data_pattern:0:-1}
data_test_null=$(printf '${data[%s]}' ${!data[@]})
if test -e $database; then
cp $database $in_ram_database
sql_cols=$(sqlite_request "pragma table_info(setxml)" | awk -F'|' '{printf("%s %s ", $2, $3)}')
script_cols="file text variantof integer "
for col in ${!data[@]}; do
script_cols+="$col ${data["$col"]} "
done
script_cols+="installed integer " # last space is important
if test "$sql_cols" != "$script_cols"; then
echo "ALERT: datbase version mismatch !"
exit 1
fi
if sqlite_request '.tables' | grep -q 'recover_'; then
echo "recovering from previous saved state"
hangar[id]=$(sqlite_request "select hangar from recover_aircrafts limit 1")
eval $(sqlite_request "select printf('hangar[name]=%s;hangar[url]=%s;hangar[type]=%s;hangar[source]=%s',
name, url, type, source)
from hangars
where id = '${hangar[id]}'")
source $(grep -l "^\s*hangar\[name\]=${hangar[name]}\s*$" ${0%*/}.d/*.hangar)
eval "getfromrepo () {$(declare -f getfromrepo | sed '1,2d;$d'); xmlremovecomments;}"
apply_revision
exit
fi
fi
sqlite_request "create table if not exists hangars (
id integer primary key,
name text,
source text,
type text,
url text,
path text,
active integer)"
sqlite_request 'create unique index if not exists "index_hangars" on hangars (url)'
sqlite_request "create table if not exists aircrafts (
id integer primary key,
name text,
revision integer,
date integer,
author text,
hangar integer)"
sqlite_request 'create unique index if not exists "index_aircrafts" on aircrafts (name, hangar)'
sqlite_request "create table if not exists setxml (
file text,
variantof integer,
$(for col in ${!data[@]}; do printf "'%s' %s, " $col ${data[$col]}; done)
installed integer)"
sqlite_request 'create unique index if not exists "index_setxml" on setxml (file, variantof)'
for file in $(find ${0%*/}.d -type f -name "*.hangar"); do
unset hangar error_message
unset -f getfromrepo parse_repo_history
declare -A hangar
source $file
test -n "${hangar[name]}" \
-a -n "${hangar[source]}" \
-a -n "${hangar[type]}" \
-a -n "${hangar[url]}" \
-a -n "${hangar[active]}" || \
error_message="${error_message:+$error_message, }missing hangar data"
declare -f getfromrepo > /dev/null || \
error_message="${error_message:+$error_message, }missing getfromrepo function"
declare -f parse_repo_history > /dev/null || \
error_message="${error_message:+$error_message, }missing parse_repo_history function"
if test -n "$error_message"; then
echo "file $file isn't a valid hangar ($error_message)"
continue
fi
sqlite_request "insert into hangars (name, source, type, url, path, active)
values (
'${hangar[name]}', '${hangar[source]}', '${hangar[type]}',
'${hangar[url]}', '${hangar[path]}', ${hangar[active]})
on conflict (url) where url = '${hangar[url]}' do
update set
name = '${hangar[name]}',
path = '${hangar[path]}',
active = ${hangar[active]}
where url = '${hangar[url]}'"
done
unset hangar
unset -f getfromrepo parse_repo_history
declare -A hangar ac_ass_array
for h_id in $(sqlite_request "select id from hangars where active = 1"); do
sqlite_request 'create table if not exists recover_aircrafts (
name text,
revision integer,
date integer,
author text,
hangar integer)'
sqlite_request 'create table if not exists recover_setxml (
ac text,
sx text)'
sqlite_request 'create unique index if not exists "index_recover_setxml" on recover_setxml (ac, sx)'
eval $(sqlite_request "select printf('hangar[id]=%i;hangar[source]=%s;', id, source)
from hangars
where id = '${h_id}'")
source $(grep -l "^\s*hangar\[source\]=${hangar[source]}\s*$" ${0%*/}.d/*.hangar)
eval "getfromrepo () {$(declare -f getfromrepo | sed '1,2d;$d'); xmlremovecomments;}"
echo -e "=${hangar[name]//?/=}=\n ${hangar[name]} \n=${hangar[name]//?/=}="
latest_revision=$(( $(sqlite_request "select max(revision)
from aircrafts inner join hangars
where hangars.id = aircrafts.hangar and hangars.name = '${hangar[name]}'") + 1 ))
parse_repo_history
if declare -f on_exit > /dev/null; then
on_exit
fi
sqlite_request "drop index 'index_recover_setxml'"
sqlite_request "drop table recover_aircrafts"
sqlite_request "drop table recover_setxml"
done