@@ -1,30 +1,75 @@

 #!/bin/bash

+set -e

+

+declare -A data=(

+        [/sim/description]=text

+        [/sim/long-description]=text

+        [/sim/author]=text

+        [/sim/flight-model]=text

+        [/sim/type]=text

+        [/sim/model/path]=text

+)

 fgaddon_url=https://sourceforge.net/p/flightgear/fgaddon/HEAD/tree/trunk/Aircraft

 fgaddon_svn=https://svn.code.sf.net/p/flightgear/fgaddon/trunk/Aircraft

 fgaddon_path=$HOME/.fgfs/flightgear-fgaddon/Aircraft

+database=${DB:-$0.db}

+#locale=fr

+

+test -r "$0.conf" && source $0.conf && echo config red

+

 aircrafts=$(mktemp --dry-run /dev/shm/Aircraft-XXXXXXXXX)

 aircraft=$(mktemp --dry-run /dev/shm/aircraft-XXXXXXX)

 setxml=$(mktemp --dry-run /dev/shm/setxml-XXXXXXXX)

 in_ram_database=$(mktemp --dry-run /dev/shm/XXXXXXX)

-database=${DB:-$0.db}

-attributes=(description long-description author flight-model type)

-xmlgetnext () {

-   local IFS='>'

-   read -d '<' TAG VALUE

-   TAG=$(tr '\n' ' ' <<< $TAG | sed 's/  */ /g; s/ *$//')

+function xmlgetnext () {

+    local IFS='>'

+    read -d '<' TAG VALUE

+    # by design, the first TAG/VALUE pair is empty

+    # to avoid infinite loops at end of file parsing we return an error

+    # the next time we find an empty TAG

+    if test -z "$TAG"; then

+        test ${xmlgetnext_firstentry:-1} -eq 1 && xmlgetnext_firstentry=0 || return 1;

+    fi

+    # process $TAG only if necessary

+    local _TAG=$(printf '%q' $TAG)

+    if test ${_TAG:0:1} = '$'; then

+        TAG=$(tr '\n' ' ' <<< $TAG | sed 's/  */ /g; s/ *$//')

+    fi

 }

 function sqlite_request () {

-    sqlite3 "$in_ram_database" <<< "$1"

+    if ! sqlite3 "$in_ram_database" <<< "$1"; then

+        register_state

+    fi

+}

+

+function xmlremovecomments () {

+    sed -ri 's/<(!--|script>)/\n&/;s/(<\/script|--)>/&\n/' $setxml

+    sed -ri '/<(script>|!--).*(<\/script|--)>/d;/<(script>|!--)/,/(<\/script|--)>/d' $setxml

+}

+

+function trap_break () {

+    trap '' INT

+    echo "stop requested"

+    register_state

 }

 function trap_exit () {

+    trapped_rc=$?

+    trap '' INT

+    rm -f $aircrafts $aircraft $setxml

+    if test ! -e $in_ram_database; then

+        exit

+    fi

+    test $trapped_rc -ne 0 && register_state

     echo "updating installation status"

-    for ac in $(sqlite_request 'select printf("%i:%s/%s", aircrafts.id, aircrafts.name, setxml.file) from aircrafts inner join setxml where aircrafts.id = setxml.variantof and setxml.installed != 0;'); do

+    for ac in $(sqlite_request 'select printf("%i:%s/%s", aircrafts.id, aircrafts.name, setxml.file)

+                                from aircrafts inner join setxml

+                                where aircrafts.id = setxml.variantof and setxml.installed != 0;'); do

         ac_path=${ac#*:}

         if test ! -e $fgaddon_path/$ac_path-set.xml; then

             sqlite_request "update setxml set installed = 0 where file = '${ac_path#*/}' and variantof = ${ac%:*}"

@@ -43,9 +88,15 @@ function trap_exit () {

         fi

         sqlite_request "update setxml set installed = $install_type

                         where exists (

-                            select 1 from aircrafts where name = '${ac/\/}' and setxml.variantof = id

+                            select 1

+                            from aircrafts

+                            where name = '${ac/\/}' and setxml.variantof = id

                         )"

     done

+    missing_setxml=$(sqlite_request "select name from aircrafts where id not in (select variantof from setxml)")

+    if test -n "$missing_setxml"; then

+        echo "missing setxml config: $missing_setxml"

+    fi

     if test -r "$database" && md5sum $in_ram_database | sed "s,$in_ram_database,$database," | md5sum --status -c -; then

         rm -f $in_ram_database

         echo "no changes in $database"

@@ -60,146 +111,317 @@ function trap_exit () {

         rm -f $in_ram_database

         echo "nothing can be done with $database !"

     fi

-    rm -f $aircrafts $aircraft $setxml

 }

-update_database () {

-    let progress++

-    echo "[ ${progress}${total:+/$total} ] $ac"

+function register_state () {

+set +x

+    sqlite_request "drop table if exists recover_rev"

+    sqlite_request "create table recover_rev (

+                        revkey text,

+                        revision integer,

+                        revauthor text,

+                        revdate integer

+                    )"

+    for revkey in ${!revision[@]}; do

+        sqlite_request "insert into recover_rev values (

+                            '$revkey',

+                            ${revision[$revkey]:-0},

+                            '${revauthor[$revkey]}',

+                            ${revdate[$revkey]:-0}

+                        )"

+    done

+    sqlite_request "drop table if exists recover_setxmlmodified"

+    sqlite_request "create table if not exists recover_setxmlmodified (

+                        sx text

+                    )"

+    for sx in ${!setxmlmodified[@]}; do

+        sqlite_request "insert into recover_setxmlmodified values (

+                            '$sx'

+                        )"

+    done

+    set +x

+    exit

+}

+

+function update_database () {

+    echo "[ ${#revision[@]} ] ${ac:1}"

-    dbupdate=$(sqlite_request "select revision from aircrafts where name is '$ac'")

+    dbupdate=$(sqlite_request "select revision from aircrafts where name is '${ac:1}'")

     if test -z "$dbupdate"; then

         sqlite_request "insert into aircrafts (name, revision, date, author)

-                        values ('$ac', ${revision[$ac]}, ${revdate[$ac]}, '${revauthor[$ac]}')"

+                        values ('${ac:1}', ${revision[$ac]}, ${revdate[$ac]}, '${revauthor[$ac]}')"

     elif test $dbupdate -lt ${revision[$ac]}; then

         sqlite_request "update aircrafts set

                             revision = ${revision[$ac]},

                             author   = '${revauthor[$ac]}',

                             date = ${revdate[$ac]}

-                        where name is '$ac'"

+                        where name is '${ac:1}'"

     fi

-    id=$(sqlite_request "select id from aircrafts where name is '$ac'")

+    id=$(sqlite_request "select id from aircrafts where name is '${ac:1}'")

-    for sx in ${setxmlmodified[$ac]}; do

-        test -n "${_sx[$ac/$sx]}" && continue

-        _sx[$ac/$sx]="already done"

-        unset description long_description author flight_model in_sim

+    for sx in ${!setxmlmodified[@]}; do

+        unset include include_rootpath

+        [[ "$sx" =~ ^"${ac:1}/" ]] || continue

+        for col in ${!data[@]}; do

+            data[$col]=

+        done

+        sx=${sx#*/}

         echo " -> $sx"

-        svn export --quiet --force --revision ${revision[$ac]} $fgaddon_svn/$ac/$sx-set.xml $setxml

+        if ! svn export --quiet --force $fgaddon_svn/${ac:1}/$sx-set.xml $setxml; then

+            register_state

+        fi

+        xmlremovecomments

+        unset xmlgetnext_firstentry property

         while xmlgetnext; do

-            if [[ "$TAG" = @($attributes_case) ]]; then

-                test -n "$in_sim" && eval "${TAG//-/_}=\"$VALUE\""

-                test -n "$description" \

-                  -a -n "$long_description" \

-                  -a -n "$author" \

-                  -a -n "$flight_model" \

-                  -a -n "$type" && break

-            elif test "$TAG" = 'sim'; then

-                in_sim=1

-            elif test "$TAG" = '/sim'; then

-                break

+            case "${TAG:0:1}" in

+                ''|'?'|'!')

+                    continue;;

+                /)

+                    property=${property%/*};;

+                *)

+                    if test "${TAG: -1}" != '/'; then

+                        property+=/${TAG%% *}

+                    fi

+#                    property+=/${TAG%% *}

+#                    if test "${TAG: -1}" = '/'; then

+#                        _TAG=${TAG:0:-1}

+#                        test "${_TAG#* }" != "${_TAG}" && eval "${_TAG#* }"

+#                        property=${property%/*}

+#                    fi;;

+            esac

+

+            if [[ "$TAG" =~ ^"PropertyList include=" ]]; then

+                include_rootpath=${include%/*}

+                test $include = $include_rootpath && unset include_rootpath

+                eval $(echo ${TAG#* })

+                [[ "$include" =~ ^Aircraft/Generic/ ]] && unset include include_rootpath && continue

+                if [[ "$include" =~ ^'../' ]]; then

+                    if test -n "$include_rootpath"; then

+                        if [[ "$include_rootpath" =~ '/' ]]; then

+                            include_rootpath=${include_rootpath%/*}

+                        else

+                            unset include_rootpath

+                        fi

+                    else

+                        ac_save=$ac

+                        unset ac

+                    fi

+                    include=${include/\.\.\/}

+                fi

+                if ! svn cat $fgaddon_svn/${ac:1}/${include_rootpath:+$include_rootpath/}$include >> $setxml; then

+                    register_state

+                fi

+                xmlremovecomments

+            fi

+

+            if [[ "$property" = /PropertyList@($data_pattern) ]]; then

+                eval "data[${property/\/PropertyList}]=\"${VALUE//\"/\\\"}\""

+                data[${property/\/PropertyList}]=$(tr '\n' ' ' <<< ${data[${property/\/PropertyList}]} | sed -r 's/^\s*//;s/\s+/ /g;s/\s*$//')

             fi

-        done < $setxml

-        test -z "$description" \

-          -a -z "$long_description" \

-          -a -z "$author" \

-          -a -z "$flight_model" \

-          -a -z "$type" && echo "WARNING: no info found, skipping" && continue

+            if test -n "$_TAG"; then

+                # _TAG non-null means xml props ends with /

+                # need to go back of last property

+                property=${property/%\/${_TAG%% *}}

+                unset _TAG

+            fi

-        if test -n "$description" -a -z "$flight_model"; then

-            grep -qi 'jsbsim' <<< "$description" && flight_model='jsb'

-            grep -qi 'yasim' <<< "$description" && flight_model='yasim'

-        fi

+            # continue parsing (while loop) until everything's found

+            for col in ${!data[@]}; do

+                test -z "${data[$col]}" && continue 2

+            done

+            break # everything's found

+        done < $setxml

-        if test -n "$long_description"; then

-            long_description=$(sed 's/^\s*//g' <<< $long_description | tr '\n' ' ')

+        if eval "test -z \"$data_test_null\""; then

+            echo "WARNING: no info found, skipping"

+            continue

         fi

         known=$(sqlite_request "select variantof from setxml where file is '$sx'")

         if test -n "$known"; then

-            for attribute in ${attributes[@]}; do

-                dbvalue=$(sqlite_request "select $attribute from setxml where file is '$sx'")

-                eval "wgvalue=\$$attribute"

-                if test "$dbvalue" != "$wgvalue" -a -n "$wgvalue"; then

-                    sqlite_request "update setxml set $attribute = '$wgvalue' where file is '$sx'"

+            for col in ${!data[@]}; do

+                dbvalue=$(sqlite_request "select '$col'

+                                          from setxml

+                                          where file is '$sx' and variantof = $known")

+                if test "$dbvalue" != "${data[$col]}" -a -n "${data[$col]}"; then

+                    sqlite_request "update setxml

+                                    set '$col' = '${data[$col]//\'/\'\'}'

+                                    where file is '$sx' and variantof = $known"

                 fi

             done

         else

-            sqlite_request "insert into setxml values ('$sx', $id, '$description', '$long_description', '$author', '$type', '$flight_model', 0)"

+            values="'$sx', $id, "

+            for col in ${!data[@]}; do

+                values+="'${data[$col]//\'/\'\'}', "

+            done

+            values+=0

+            sqlite_request "insert into setxml values ($values)"

+        fi

+        test -n "$ac_save" && ac=$ac_save

+        unset setxmlmodified[${ac:1}/$sx]

+    done

+    unset revision[$ac]

+}

+

+function apply_revision () {

+    for ac in "${!revision[@]}"; do

+        update_database

+        if test -d $fgaddon_path/${ac:1}/.svn \

+        && test "$(svn info --show-item=url $fgaddon_path/${ac:1})" != "$fgaddon_svn/${ac:1}" \

+        || test -d $fgaddon_path/${ac:1} -a ! -d $fgaddon_path/${ac:1}/.svn; then

+            echo "INFO: local ${ac:1} installed out from repo" >&2

         fi

     done

 }

+trap trap_break INT

 trap trap_exit EXIT

-test -e $database && cp $database $in_ram_database

+stty -echoctl

+

+declare -A revision revauthor revdate setxmlmodified files revpath

+data_pattern=$(printf "%s|" ${!data[@]})

+data_pattern=${data_pattern:0:-1}

+data_test_null=$(printf '${data[%s]}' ${!data[@]})

+

+if test -e $database; then

+    cp $database $in_ram_database

+    sql_cols=$(sqlite_request "pragma table_info(setxml)" | awk -F'|' '{printf("%s %s ", $2, $3)}')

+    script_cols="file text variantof integer "

+    for col in ${!data[@]}; do

+        script_cols+="$col ${data["$col"]} "

+    done

+    script_cols+="installed integer " # last space is important

+    if test "$sql_cols" != "$script_cols"; then

+        echo "ALERT: datbase version mismatch !"

+        exit 1

+    fi

+    if sqlite_request '.tables' | grep -q 'recover_' && test -z "$1"; then

+        echo "recovering from previous saved state"

+        eval $(sqlite_request "select printf('revision[%s]=%u;revauthor[%s]=%s;revdate[%s]=%u;',

+                                       revkey, revision,

+                                       revkey, revauthor,

+                                       revkey, revdate)

+                               from recover_rev")

+        eval $(sqlite_request "select printf('setxmlmodified[%s]=1;', sx)

+                               from recover_setxmlmodified")

+        sqlite_request "drop table recover_rev"

+        sqlite_request "drop table recover_setxmlmodified"

+        apply_revision

+        exit

+    fi

+fi

+

+sqlite_request "create table if not exists aircrafts (

+                    id integer primary key,

+                    name text,

+                    revision integer,

+                    date integer,

+                    author text)"

-sqlite_request "create table if not exists aircrafts (id integer primary key, name text, revision integer, date integer, author text)"

-sqlite_request "create table if not exists setxml (file text, variantof integer, description text, long_description text, author text, type text, flight_model text, installed integer)"

+sqlite_request "create table if not exists setxml (

+                    file text,

+                    variantof integer,

+                    $(for col in ${!data[@]}; do printf "'%s' %s, " $col ${data[$col]}; done)

+                    installed integer)"

-attributes_case=$(printf "%s|" "${attributes[@]}")

-attributes_case=${attributes_case:0:-1}

 latest_revision=$(sqlite_request "select max(revision) from aircrafts")

+# for debugging purpose

+if test -n "$2"; then

+    ac=_${1%/*}

+    revision[$ac]=1

+    revdate[$ac]=0

+    revauthor[$ac]=foobar

+    setxmlmodified[${ac:1}/${1#*/}]=1

+    set -x

+    update_database

+    set +x

+    exit

+elif test -n "$1"; then

+    ac=_${1%/*}

+    eval $(sqlite_request "select printf('revision[_%s]=%s;revdate[_%s]=%i;revauthor[_%s]=%s;',

+                                            name, revision,

+                                            name, date,

+                                            name, author)

+                           from aircrafts

+                           where name = '${ac:1}'")

+    setxmlmodified[${ac:1}/${1#*/}]=1

+    if test -z "${revision[$ac]}"; then

+        echo "aircraft ${ac:1} not found"

+        rm $in_ram_database

+        exit

+    fi

+    update_database

+    exit

+fi

+

 echo "downloading FGADDON history from revision ${latest_revision:-0}"

 svn log --revision ${latest_revision:-0}:HEAD --xml --verbose $fgaddon_svn > $aircrafts

+total=$(grep -c '<logentry' $aircrafts)

+progress=0

+

+echo parsing history

+

 while xmlgetnext; do

     case "$TAG" in

         'logentry revision='*)

             eval $(echo ${TAG#* })

-            unset revdate revauthor ac path_list setxmlmodified

+            for action in ${!revpath[@]}; do

+                unset revpath[$action]

+            done

         ;;

         'author')

-            revauthor=$VALUE

+            revauthor=${VALUE//\'/\'\'}

         ;;

         'date')

             revdate=$(date +%s -d "$VALUE")

         ;;

         'path '*)

-            path_list[${#path_list[@]}]="$VALUE"

+            TAG=${TAG#* }

+            TAG=${TAG// /;}

+            TAG=${TAG//-/_}

+            eval $(echo ${TAG// /;})

+            path=(${VALUE//\// })

+            if test $kind = 'file' -a ${#path[@]} -gt 3; then

+                revpath[$action]+="$VALUE "

+            elif test $kind = 'dir' -a ${#path[@]} -eq 3 -a $action = 'D'; then

+                files[_${path[2]}]=0

+                unset revision[_${path[2]}] revauthor[_${path[2]}] revdate[_${path[2]}]

+                for sx in ${!setxmlmodified[@]}; do

+                    [[ "$sx" =~ "${path[2]}/" ]] && unset setxmlmodified[$sx]

+                done

+            fi

         ;;

         '/logentry')

-            revlog[${#revlog[@]}]=$(

-                printf '%s\n' "${path_list[@]}" | awk -F/ -v revision=$revision -v revauthor=$revauthor -v revdate=$revdate -v dq='"' '

-                {

-                    a[$4]++

-                    if ($5 ~ "-set.xml$") b[$4]=b[$4] " " gensub("-set.xml$", "", "1", $5)

-                }

-                END {

-                    for (i in a)

-                        printf("<revision[%s]=%i;revauthor[%s]=%s;revdate[%s]=%i;%s>",

-                                i, revision,

-                                i, revauthor,

-                                i, revdate,

-                                b[i] != "" ? "setxmlmodified[" i "]=" dq gensub("^ ", "", "g", b[i]) " ${setxmlmodified[" i "]}" dq ";" : "")

-                }')

-        ;;

-        '/log')

-            unset revision revauthor revdate setxmlmodified

-            declare -A revision revauthor revdate setxmlmodified

-            analyse () {

-                while xmlgetnext; do

-                    test -z "$TAG" -a ${#revision[@]} -eq 0 && continue

-                    case $TAG in

-                        'BEGIN') continue;;

-                        'END'  ) break 2;;

-                    esac

-                    echo $TAG

-                done <<< "<BEGIN>${revlog[@]}<END>"

-            }

-            eval $(analyse)

-            total=${#revision[@]}

-            declare -A _sx

-            for ac in "${!revision[@]}"; do

-                update_database

-                if test -d $fgaddon_path/$ac/.svn \

-                && test "$(svn info --show-item=url $fgaddon_path/$ac)" != "$fgaddon_svn/$ac" \

-                || test -d $fgaddon_path/$ac -a ! -d $fgaddon_path/$ac/.svn; then

-                    echo "INFO: local $ac installed out from repo" >&2

+            for item in ${revpath[D]}; do

+                path=(${item//\// })

+                [[ "${path[3]}" =~ "-set.xml" ]] && unset setxmlmodified[${path[2]}/${path[3]/-set.xml}]

+                files[_${path[2]}]=$(( --files[_${path[2]}] ))

+                if test ${files[_${path[2]}]} -le 0; then

+                    unset revision[_${path[2]}] revauthor[_${path[2]}] revdate[_${path[2]}]

                 fi

             done

+            for action in A M R; do

+                for item in ${revpath[$action]}; do

+                    path=(${item//\// })

+                    revision[_${path[2]}]=$revision

+                    revauthor[_${path[2]}]=$revauthor

+                    revdate[_${path[2]}]=$revdate

+                    [[ "${path[3]}" =~ "-set.xml" ]] && setxmlmodified[${path[2]}/${path[3]/-set.xml}]=1

+                    test $action = 'A' && files[_${path[2]}]=$(( ++files[_${path[2]}] ))

+                done

+            done

+            newprogress=$((++logentry * 100 / $total))

+            if test $(( $newprogress - $progress )) -ge ${progress_granularity:-1}; then

+                progress=$newprogress

+                echo "$progress% (${#revision[@]})"

+            fi

+        ;;

+        '/log')

+            apply_revision

             break

         ;;

     esac

@@ -157,9 +157,8 @@ function fgfs () {

             elif which sqlite3 > /dev/null 2>&1 \

             && test -r $FGADDON/fgaddon.db \

             && test $(sqlite3 $FGADDON/fgaddon.db <<< "select count(file) from setxml where file is '${fgfs_arg}'") -eq 1; then

-                model=$(sqlite3 $FGADDON/fgaddon.db <<< "select name from aircrafts inner join setxml on aircrafts.id = setxml.variantof where setxml.file is '${fgfs_arg}-set.xml'")

+                model=$(sqlite3 $FGADDON/fgaddon.db <<< "select name from aircrafts inner join setxml on aircrafts.id = setxml.variantof where setxml.file is '${fgfs_arg}'")

                 read -q "REPLY?download $model ? (y/N) "

-                read

                 if test -n "$REPLY" && test ${REPLY:l} = "y"; then

                     svn co https://svn.code.sf.net/p/flightgear/fgaddon/trunk/Aircraft/$model $FGADDON/Aircraft/$model

                     fgfs_args+=("--aircraft=$fgfs_arg")