Sébastien MARQUE améliore test de version
256ab7b a year ago
1 contributor
519 lines | 20.795kb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520#!/bin/bash

set -e

declare -A datatypes=(
        [/sim/description]=text
        [/sim/long-description]=text
        [/sim/author]=text
        [/sim/flight-model]=text
        [/sim/type]=text
        [/sim/model/path]=text
        [/sim/rating/FDM]="integer DEFAULT 0"
        [/sim/rating/systems]="integer DEFAULT 0"
        [/sim/rating/cockpit]="integer DEFAULT 0"
        [/sim/rating/model]="integer DEFAULT 0"
)

missing_data_check=( /sim/model/path )

database=${DB:-$0.db}
test -r "$0.conf" && source "$0.conf"

#locale=fr

tempid=$(mktemp --dry-run XXXXXXX)
temppath=/dev/shm

setxml=$temppath/setxml-$tempid
json_file=$temppath/github_json-$tempid
in_ram_database=$temppath/${database##*/}-$tempid

function xmlgetnext () {
    local IFS='>'
    read -d '<' TAG VALUE
    # by design, the first TAG/VALUE pair is empty
    # to avoid infinite loops at end of file parsing we return an error
    # the next time we find an empty TAG
    if test -z "$TAG"; then
        test ${xmlgetnext_empty_tag:-0} -gt 0 && return 1
        xmlgetnext_empty_tag=$(( xmlgetnext_empty_tag + 1 ))
    fi
    # process $TAG only if necessary
    local _TAG=$(printf '%q' $TAG)
    if test ${_TAG:0:1} = '$'; then
        TAG=$(tr '\n' ' ' <<< $TAG | sed 's/  */ /g; s/ *$//')
    fi
}

function json () {
    jq --raw-output "$1" < ${2:-${json_file:?}}
}

rm -f $temppath/sqlite_request
function sqlite_request () {
    echo -e "## REQ $(( ++sqlite_request_count ))\n${1}\n" >> $temppath/sqlite_request
    sqlite3 "$in_ram_database" <<< "$1"
}

function xmlremovecomments () {
    sed -ri 's/<(!--|script>)/\n&/;s/(<\/script|--)>/&\n/' $setxml
    sed -ri '/<(script>|!--).*(<\/script|--)>/d;/<(script>|!--)/,/(<\/script|--)>/d' $setxml
    sed -i 's/\xef\xbb\xbf//;s/\r//' $setxml # removes BOM and ^M
}

function trap_break () {
    trap '' INT
    echo "stop requested"
}

function trap_exit () {
    trapped_rc=$?
    trap '' INT

    if declare -f on_exit > /dev/null; then
        on_exit
    fi

    if test ! -e $in_ram_database; then
        exit
    fi
    echo "updating installation status"
    for ac in $(sqlite_request 'select printf("%i:%s/%s", aircrafts.id, aircrafts.name, setxml.file)
                                from aircrafts inner join setxml
                                where aircrafts.id = setxml.variantof and setxml.installed != 0;'); do
        ac_path=${ac#*:}
        if test ! -e ${hangar[path]}/$ac_path-set.xml; then
            sqlite_request "update setxml set installed = 0 where file = '${ac_path#*/}' and variantof = ${ac%:*}"
        fi
    done
    for ac in ${hangar[path]}/*/*-set.xml; do
        ac=${ac/${hangar[path]}}
        sx=${ac##*/}
        ac=${ac%/*}
        if test -d ${hangar[path]}/$ac/.svn; then
            install_type=1
        elif test -d ${hangar[path]}/$ac/.git; then
            install_type=2
        else
            install_type=3
        fi
        sqlite_request "update setxml set installed = $install_type
                        where exists (
                            select 1
                            from aircrafts
                            where name = '${ac/\/}' and setxml.variantof = id
                        )"
    done
    local missing_setxml=$(sqlite_request "select printf(' - %s (%s)', aircrafts.name, hangars.name)
                                     from aircrafts inner join hangars
                                     where hangars.id = aircrafts.hangar and aircrafts.id not in (select variantof from setxml)")
    if test -n "$missing_setxml"; then
        echo -e "missing setxml config for :\n$missing_setxml"
    fi

    for data_presence_check in ${missing_data_check[@]}; do
        if [[ -v datatypes[$data_presence_check] ]]; then
            local missing_data=$(sqlite_request "select count(setxml.file)
                                                 from aircrafts inner join setxml
                                                 where aircrafts.id = setxml.variantof and setxml.\`$data_presence_check\` = ''")
            if test $missing_data -gt 0; then
                echo "$missing_data aircrafts without $data_presence_check information"
                if test $missing_data -le 10; then
                    echo "aircrafts without $data_presence_check information:"
                    sqlite_request "select printf(' - %s/%s (%s)', aircrafts.name, setxml.file, hangars.name)
                                    from aircrafts inner join setxml, hangars
                                    where
                                        aircrafts.id = setxml.variantof
                                    and
                                        aircrafts.hangar = hangars.id
                                    and
                                        setxml.\`$data_presence_check\` = ''"
                fi
            fi
        fi
    done

    if test -r "$database" && sqldiff=$(sqldiff $in_ram_database $database) && test -z "$sqldiff"; then
        echo "no changes in $database"
    elif test -w "$database"; then
        rm -f "$database"
        sqlite_request '.dump' | sqlite3 "$database"
        echo "database $database updated"
    elif test ! -e "$database" -a -w ${database%/*}; then
        sqlite_request '.dump' | sqlite3 "$database"
        echo "database $database created"
    else
        echo "nothing can be done with $database !"
    fi
    find $temppath -type f -name "*-$tempid" -delete
}

function update_database () {
    sqlite_request "insert into aircrafts (name, author, revision, date, hangar)
                    select  name, author, revision, date, hangar from recover_aircrafts
                        where recover_aircrafts.name = '$ac' and recover_aircrafts.hangar = ${hangar[id]}
                    on conflict (name, hangar) where aircrafts.name = '$ac' and aircrafts.hangar = ${hangar[id]} do
                        update set
                            author   = (select author   from recover_aircrafts where name = '$ac'),
                            revision = (select revision from recover_aircrafts where name = '$ac'),
                            date     = (select date     from recover_aircrafts where name = '$ac')
                        where aircrafts.name = '$ac' and aircrafts.hangar = ${hangar[id]}"

    id=$(sqlite_request "select id from aircrafts where name is '${ac}' and hangar = ${hangar[id]}")

    echo $(sqlite_request "select printf('[ %i/%i ] $ac', count(sx), count(distinct ac)) from recover_setxml")

    for sx in $(sqlite_request "select distinct sx from recover_setxml where ac = '$ac'"); do
        unset data
        declare -A data

        printf " -> $sx"
        getfromrepo ${ac}/$sx-set.xml > $setxml

        unset xmlgetnext_empty_tag property include include_rootpath ac_save
        while xmlgetnext; do
            if [[ "$TAG" =~ ^"PropertyList include=" ]]; then
                include_rootpath=${include%/*}
                test $include = $include_rootpath && unset include_rootpath

                eval $(echo ${TAG#* }) # include="..."

                if [[ "$include" =~ ^Aircraft/Generic/ ]]; then
                    unset include include_rootpath
                    continue

                elif [[ "$include" =~ ^'../' ]]; then
                    if test -n "$include_rootpath"; then
                        if [[ "$include_rootpath" =~ '/' ]]; then
                            include_rootpath=${include_rootpath%/*}
                        else
                            unset include_rootpath
                        fi
                    else
                        ac_save=$ac
                        unset ac
                    fi
                    include=${include/\.\.\/}
                fi
                getfromrepo ${ac}/${include_rootpath:+$include_rootpath/}$include >> $setxml
            fi
        done < $setxml

        test -n "$ac_save" && ac=$ac_save

# some aircrafts (mostly from the helijah's files architecture template)
# break because of infinite loop in middle of file
# I can't find the reason of this infinite loop
# this is the reason of this double-pass
        unset xmlgetnext_empty_tag property
        while xmlgetnext; do
        case "${TAG:0:1}" in
                ''|'?'|'!')
                    continue;;
                /)
                    property=${property%/*};;
                *)
                    if test "${TAG: -1}" != '/'; then
                        property+=/${TAG%% *}
                    fi;;
            esac

            if [[ "$property" = /PropertyList@($data_pattern) ]]; then
                if test -z "${data[${property/\/PropertyList}]}"; then
                    eval "data[${property/\/PropertyList}]=\"${VALUE//\"/\\\"}\""
                    data[${property/\/PropertyList}]=$(tr '\n' ' ' <<< ${data[${property/\/PropertyList}]} | sed -r 's/^\s*//;s/\s+/ /g;s/\s*$//')
                fi
            fi

            # continue parsing (while loop) until everything's found
            for col in ${!datatypes[@]}; do
                test -z "${data[$col]}" && continue 2
            done
            break # everything's found
        done < $setxml

        if eval "test -z \"$data_test_null\""; then
            printf "\nWARNING: no info found, skipping\n"
            mkdir -p $temppath/no-data-ac
            cp -f $setxml $temppath/no-data-ac/${ac}-${sx}
        else
            insert_values="'$sx', $id, "
            insert_col='file, variantof, '
            update_values=''
            for col in ${!data[@]}; do
                if test ${datatypes[$col]%% *} = 'text'; then
                    single_quote="'"
                elif [[ ${datatypes[$col]%% *} = 'integer' && "${data[$col]// }" = +([0-9]) ]]; then
                    single_quote=""
                else
                    unset datatypes[$col]
                    continue
                fi
                insert_col+="\`$col\`, "
                insert_values+="$single_quote${data[$col]//\'/\'\'}$single_quote, "
                if test -n "${data[$col]}"; then
                    update_values+="\`$col\` = $single_quote${data[$col]//\'/\'\'}$single_quote, "
                fi
            done
            local flag_new=
            local flag_status=
            if test $(sqlite_request "select count(file) from setxml where file = '$sx' and variantof = $id") -eq 0; then
                flag_new="NEW"
            fi
            for criteria in FDM model systems cockpit; do
                if test ${data[/sim/rating/$criteria]:-0} -ge 4; then
                    flag_status+='*'
                fi
            done
            if test -n "$flag_new" -o -n "$flag_status"; then
                printf " (${flag_new:+$flag_new }$flag_status)"
            fi
            printf "\n"
            sqlite_request "insert into setxml (${insert_col%,*}, installed) values (${insert_values%,*}, 0)
                            on conflict (file, variantof) where file = '$sx' and variantof = $id do
                                update set
                                    ${update_values%,*}, installed = 0
                                where
                                    file = '$sx' and variantof = $id"
        fi

        sqlite_request "delete from recover_setxml where ac = '$ac' and sx = '$sx'"
    done
}

function add_record () {
    ac_ass_array[$1]="$2"
}

function get_record () {
    if test -n "$1"; then
        echo "${ac_ass_array[$1]}"
    else
        for k in ${!ac_ass_array[@]}; do
            echo $k = ${ac_ass_array[$k]}
        done
    fi
}

function add_aircraft () {
    for key in name revision date author; do
        test -n "${ac_ass_array[$key]}" # exit if missing data (with the help of "set -e")
    done
    local new_revision=$(sqlite_request "select revision from recover_aircrafts
                                         where name = '${ac_ass_array[name]}'")
    if test -z "${new_revision}"; then
        sqlite_request "insert into recover_aircrafts (name, revision, date, author, hangar)
                            values (
                            '${ac_ass_array[name]}',
                             ${ac_ass_array[revision]},
                             ${ac_ass_array[date]},
                            '${ac_ass_array[author]}',
                             ${hangar[id]})"
    elif test ${new_revision} -lt ${ac_ass_array[revision]//\"}; then
        sqlite_request "update recover_aircrafts
                        set
                            revision =  ${ac_ass_array[revision]},
                            date     =  ${ac_ass_array[date]},
                            author   = '${ac_ass_array[author]}',
                            hangar   =  ${hangar[id]}
                        where name = '${ac_ass_array[name]}'"
    fi
    for key in name revision date author; do
        ac_ass_array[$key]=''
    done
}

function add_setxml_for_aircraft () {
    sqlite_request "insert into recover_setxml values ('$1', '${2/%-set.xml}')
                    on conflict (ac, sx) where ac = '$1' and sx = '${2/%-set.xml}'
                    do nothing"
}

function apply_revision () {
    for ac in $(sqlite_request "select name from recover_aircrafts"); do
        # delete aircrafts that have been deleted from the repo
        sqlite_request "delete from setxml
                        where (file, variantof) in (
                            select file, variantof from setxml
                              inner join aircrafts
                              where aircrafts.id = setxml.variantof
                                and aircrafts.name = '$ac'
                                and aircrafts.hangar = ${hangar[id]}
                                and setxml.file not in (
                                    select sx from recover_setxml where ac = '$ac'
                                )
                            )"

        # delete aircrafts without setxml found
        sqlite_request "delete from recover_aircrafts
                        where name not in (select distinct ac from recover_setxml)"

        update_database
        if test -d ${hangar[path]}/${ac}/.${hangar[type]} \
        && \
        case ${hangar[type]} in
           svn) test "$(svn info --show-item=url ${hangar[path]}/${ac})" != "${hangar[url]}/${ac}";;
           git) test "$(git -C ${hangar[path]}/${ac} config --get remote.origin.url)" != "${hangar[url]}/${ac}.git";;
        esac \
        || test -d ${hangar[path]}/${ac} -a ! -d ${hangar[path]}/${ac}/.${hangar[type]}; then
            echo "INFO: local ${ac} installed out from repo" >&2
        fi
        sqlite_request "delete from recover_aircrafts where name = '$ac'"
    done
}

trap trap_break INT
trap trap_exit EXIT

stty -echoctl

declare -A hangar
data_pattern=$(printf "%s|" ${!datatypes[@]})
data_pattern=${data_pattern:0:-1}
data_test_null=$(printf '${data[%s]}' ${!datatypes[@]})

if test -e $database; then
    cp $database $in_ram_database

    sql_cols=$(sqlite_request "pragma table_info(setxml)" \
             | awk -F'|' '{print $2 " " tolower($3)}' \
             | sort)
    datatypes[file]=text
    datatypes[variantof]=integer
    datatypes[installed]=integer
    script_cols=$(for col in ${!datatypes[@]}; do echo "$col ${datatypes["$col"]%% *}"; done \
                | sort)
    unset datatypes[file] datatypes[variantof] datatypes[installed]
    if test "$(md5sum <<< $sql_cols)" != "$(md5sum <<< $script_cols)"; then
        echo "ALERT: datbase version mismatch !"
        echo "DB:     $sql_cols"
        echo "script: $script_cols"
        exit 1
    fi
    if sqlite_request '.tables' | grep -q 'recover_'; then
        hangar[id]=$(sqlite_request "select hangar from recover_aircrafts limit 1")
        if test -n "${hangar[id]}"; then
            echo "recovering from previous saved state"
            eval $(sqlite_request "select printf('hangar[name]=%s;hangar[url]=%s;hangar[type]=%s;hangar[source]=%s',
                                                  name,           url,           type,           source)
                                   from hangars
                                   where id = '${hangar[id]}'")
            source $(grep -l "^\s*hangar\[name\]=${hangar[name]}\s*$" ${0%*/}.d/*.hangar)
            eval "getfromrepo () {$(declare -f getfromrepo | sed '1,2d;$d'); xmlremovecomments;}"
            apply_revision
            exit
        else
            sqlite_request 'drop table recover_aircrafts'
            sqlite_request 'drop table recover_setxml'
        fi
    fi
fi

sqlite_request "create table if not exists hangars (
                    id     integer primary key,
                    name   text,
                    source text,
                    type   text,
                    url    text,
                    path   text,
                    active integer)"

sqlite_request 'create unique index if not exists "index_hangars" on hangars (url)'

sqlite_request "create table if not exists aircrafts (
                    id       integer primary key,
                    name     text,
                    revision integer,
                    date     integer,
                    author   text,
                    hangar   integer)"

sqlite_request 'create unique index if not exists "index_aircrafts" on aircrafts (name, hangar)'

sqlite_request "create table if not exists setxml (
                    file text,
                    variantof integer,
                    $(for col in ${!datatypes[@]}; do printf "'%s' %s, " $col "${datatypes[$col]}"; done)
                    installed integer)"

sqlite_request 'create unique index if not exists "index_setxml" on setxml (file, variantof)'

for file in $(find ${0%*/}.d -type f -name "*.hangar"); do
    unset hangar error_message
    unset -f getfromrepo parse_repo_history
    declare -A hangar
    source $file

    test -n "${hangar[name]}"      \
      -a -n "${hangar[source]}"    \
      -a -n "${hangar[type]}"      \
      -a -n "${hangar[url]}"       \
      -a -n "${hangar[active]}" || \
        error_message="${error_message:+$error_message, }missing hangar data"

    declare -f getfromrepo > /dev/null || \
        error_message="${error_message:+$error_message, }missing getfromrepo function"

    declare -f parse_repo_history > /dev/null || \
        error_message="${error_message:+$error_message, }missing parse_repo_history function"

    if test -n "$error_message"; then
        echo "file $file isn't a valid hangar ($error_message)"
        continue
    fi

    sqlite_request "insert into hangars (name, source, type, url, path, active)
                    values (
                        '${hangar[name]}', '${hangar[source]}', '${hangar[type]}',
                        '${hangar[url]}',  '${hangar[path]}',    ${hangar[active]})
                    on conflict (url) where url = '${hangar[url]}' do
                    update set
                        name = '${hangar[name]}',
                        path = '${hangar[path]}',
                        active = ${hangar[active]}
                    where url = '${hangar[url]}'"
done

unset hangar
unset -f getfromrepo parse_repo_history
declare -A hangar ac_ass_array
for h_id in $(sqlite_request "select id from hangars where active = 1"); do

    sqlite_request 'create table if not exists recover_aircrafts (
                        name     text,
                        revision integer,
                        date     integer,
                        author   text,
                        hangar   integer)'

    sqlite_request 'create table if not exists recover_setxml (
                        ac text,
                        sx text)'

    sqlite_request 'create unique index if not exists "index_recover_setxml" on recover_setxml (ac, sx)'

    eval $(sqlite_request "select printf('hangar[id]=%i;hangar[source]=%s;', id, source)
                           from hangars
                           where id = '${h_id}'")

    source $(grep -l "^\s*hangar\[source\]=${hangar[source]}\s*$" ${0%*/}.d/*.hangar)

    eval "getfromrepo () {$(declare -f getfromrepo | sed '1,2d;$d'); xmlremovecomments;}"

    echo -e "=${hangar[name]//?/=}=\n ${hangar[name]} \n=${hangar[name]//?/=}="

    latest_revision=$(( $(sqlite_request "select max(revision)
                                          from aircrafts inner join hangars
                                          where hangars.id = aircrafts.hangar and hangars.name = '${hangar[name]}'") + 1 ))

    parse_repo_history

    if declare -f on_exit > /dev/null; then
        on_exit
    fi
    sqlite_request "drop index 'index_recover_setxml'"
    sqlite_request "drop table recover_aircrafts"
    sqlite_request "drop table recover_setxml"
done