Newer Older
519 lines | 20.766kb
déplacement depuis dépôt con...
Sébastien MARQUE authored on 2021-05-28
1
#!/bin/bash
2

            
3
set -e
4

            
5
declare -A datatypes=(
6
        [/sim/description]=text
7
        [/sim/long-description]=text
8
        [/sim/author]=text
9
        [/sim/flight-model]=text
10
        [/sim/type]=text
11
        [/sim/model/path]=text
12
        [/sim/rating/FDM]="integer DEFAULT 0"
13
        [/sim/rating/systems]="integer DEFAULT 0"
14
        [/sim/rating/cockpit]="integer DEFAULT 0"
15
        [/sim/rating/model]="integer DEFAULT 0"
16
)
17

            
18
missing_data_check=( /sim/model/path )
19

            
20
database=${DB:-$0.db}
21
test -r "$0.conf" && source "$0.conf"
22

            
23
#locale=fr
24

            
25
tempid=$(mktemp --dry-run XXXXXXX)
26
temppath=/dev/shm
27

            
28
setxml=$temppath/setxml-$tempid
29
json_file=$temppath/github_json-$tempid
30
in_ram_database=$temppath/${database##*/}-$tempid
31

            
32
function xmlgetnext () {
33
    local IFS='>'
34
    read -d '<' TAG VALUE
35
    # by design, the first TAG/VALUE pair is empty
36
    # to avoid infinite loops at end of file parsing we return an error
37
    # the next time we find an empty TAG
38
    if test -z "$TAG"; then
39
        test ${xmlgetnext_empty_tag:-0} -gt 0 && return 1
40
        xmlgetnext_empty_tag=$(( xmlgetnext_empty_tag + 1 ))
41
    fi
42
    # process $TAG only if necessary
43
    local _TAG=$(printf '%q' $TAG)
44
    if test ${_TAG:0:1} = '$'; then
45
        TAG=$(tr '\n' ' ' <<< $TAG | sed 's/  */ /g; s/ *$//')
46
    fi
47
}
48

            
49
function json () {
50
    jq --raw-output "$1" < ${2:-${json_file:?}}
51
}
52

            
53
rm -f $temppath/sqlite_request
54
function sqlite_request () {
55
    echo -e "## REQ $(( ++sqlite_request_count ))\n${1}\n" >> $temppath/sqlite_request
56
    sqlite3 "$in_ram_database" <<< "$1"
57
}
58

            
59
function xmlremovecomments () {
60
    sed -ri 's/<(!--|script>)/\n&/;s/(<\/script|--)>/&\n/' $setxml
61
    sed -ri '/<(script>|!--).*(<\/script|--)>/d;/<(script>|!--)/,/(<\/script|--)>/d' $setxml
62
    sed -i 's/\xef\xbb\xbf//;s/\r//' $setxml # removes BOM and ^M
63
}
64

            
65
function trap_break () {
66
    trap '' INT
67
    echo "stop requested"
68
}
69

            
70
function trap_exit () {
71
    trapped_rc=$?
72
    trap '' INT
73

            
74
    if declare -f on_exit > /dev/null; then
75
        on_exit
76
    fi
77

            
78
    if test ! -e $in_ram_database; then
79
        exit
80
    fi
81
    echo "updating installation status"
82
    for ac in $(sqlite_request 'select printf("%i:%s/%s", aircrafts.id, aircrafts.name, setxml.file)
83
                                from aircrafts inner join setxml
84
                                where aircrafts.id = setxml.variantof and setxml.installed != 0;'); do
85
        ac_path=${ac#*:}
86
        if test ! -e ${hangar[path]}/$ac_path-set.xml; then
87
            sqlite_request "update setxml set installed = 0 where file = '${ac_path#*/}' and variantof = ${ac%:*}"
88
        fi
89
    done
90
    for ac in ${hangar[path]}/*/*-set.xml; do
91
        ac=${ac/${hangar[path]}}
92
        sx=${ac##*/}
93
        ac=${ac%/*}
94
        if test -d ${hangar[path]}/$ac/.svn; then
95
            install_type=1
96
        elif test -d ${hangar[path]}/$ac/.git; then
97
            install_type=2
98
        else
99
            install_type=3
100
        fi
101
        sqlite_request "update setxml set installed = $install_type
102
                        where exists (
103
                            select 1
104
                            from aircrafts
105
                            where name = '${ac/\/}' and setxml.variantof = id
106
                        )"
107
    done
108
    local missing_setxml=$(sqlite_request "select printf(' - %s (%s)', aircrafts.name, hangars.name)
109
                                     from aircrafts inner join hangars
110
                                     where hangars.id = aircrafts.hangar and aircrafts.id not in (select variantof from setxml)")
111
    if test -n "$missing_setxml"; then
112
        echo -e "missing setxml config for :\n$missing_setxml"
113
    fi
114

            
115
    for data_presence_check in ${missing_data_check[@]}; do
116
        if [[ -v datatypes[$data_presence_check] ]]; then
117
            local missing_data=$(sqlite_request "select count(setxml.file)
118
                                                 from aircrafts inner join setxml
119
                                                 where aircrafts.id = setxml.variantof and setxml.\`$data_presence_check\` = ''")
120
            if test $missing_data -gt 0; then
121
                echo "$missing_data aircrafts without $data_presence_check information"
122
                if test $missing_data -le 10; then
123
                    echo "aircrafts without $data_presence_check information:"
124
                    sqlite_request "select printf(' - %s/%s (%s)', aircrafts.name, setxml.file, hangars.name)
125
                                    from aircrafts inner join setxml, hangars
126
                                    where
127
                                        aircrafts.id = setxml.variantof
128
                                    and
129
                                        aircrafts.hangar = hangars.id
130
                                    and
131
                                        setxml.\`$data_presence_check\` = ''"
132
                fi
133
            fi
134
        fi
135
    done
136

            
améliore la vérification de ...
Sébastien MARQUE authored on 2021-10-23
137
    if test -r "$database" && sqldiff=$(sqldiff $in_ram_database $database) && test -z "$sqldiff"; then
déplacement depuis dépôt con...
Sébastien MARQUE authored on 2021-05-28
138
        echo "no changes in $database"
139
    elif test -w "$database"; then
140
        rm -f "$database"
141
        sqlite_request '.dump' | sqlite3 "$database"
142
        echo "database $database updated"
143
    elif test ! -e "$database" -a -w ${database%/*}; then
144
        sqlite_request '.dump' | sqlite3 "$database"
145
        echo "database $database created"
146
    else
147
        echo "nothing can be done with $database !"
148
    fi
149
    find $temppath -type f -name "*-$tempid" -delete
150
}
151

            
152
function update_database () {
153
    sqlite_request "insert into aircrafts (name, author, revision, date, hangar)
154
                    select  name, author, revision, date, hangar from recover_aircrafts
155
                        where recover_aircrafts.name = '$ac' and recover_aircrafts.hangar = ${hangar[id]}
156
                    on conflict (name, hangar) where aircrafts.name = '$ac' and aircrafts.hangar = ${hangar[id]} do
157
                        update set
158
                            author   = (select author   from recover_aircrafts where name = '$ac'),
159
                            revision = (select revision from recover_aircrafts where name = '$ac'),
160
                            date     = (select date     from recover_aircrafts where name = '$ac')
161
                        where aircrafts.name = '$ac' and aircrafts.hangar = ${hangar[id]}"
162

            
163
    id=$(sqlite_request "select id from aircrafts where name is '${ac}' and hangar = ${hangar[id]}")
164

            
165
    echo $(sqlite_request "select printf('[ %i/%i ] $ac', count(sx), count(distinct ac)) from recover_setxml")
166

            
167
    for sx in $(sqlite_request "select distinct sx from recover_setxml where ac = '$ac'"); do
168
        unset data
169
        declare -A data
170

            
171
        printf " -> $sx"
172
        getfromrepo ${ac}/$sx-set.xml > $setxml
173

            
174
        unset xmlgetnext_empty_tag property include include_rootpath ac_save
175
        while xmlgetnext; do
176
            if [[ "$TAG" =~ ^"PropertyList include=" ]]; then
177
                include_rootpath=${include%/*}
178
                test $include = $include_rootpath && unset include_rootpath
179

            
180
                eval $(echo ${TAG#* }) # include="..."
181

            
182
                if [[ "$include" =~ ^Aircraft/Generic/ ]]; then
183
                    unset include include_rootpath
184
                    continue
185

            
186
                elif [[ "$include" =~ ^'../' ]]; then
187
                    if test -n "$include_rootpath"; then
188
                        if [[ "$include_rootpath" =~ '/' ]]; then
189
                            include_rootpath=${include_rootpath%/*}
190
                        else
191
                            unset include_rootpath
192
                        fi
193
                    else
194
                        ac_save=$ac
195
                        unset ac
196
                    fi
197
                    include=${include/\.\.\/}
198
                fi
199
                getfromrepo ${ac}/${include_rootpath:+$include_rootpath/}$include >> $setxml
200
            fi
201
        done < $setxml
202

            
203
        test -n "$ac_save" && ac=$ac_save
204

            
205
# some aircrafts (mostly from the helijah's files architecture template)
206
# break because of infinite loop in middle of file
207
# I can't find the reason of this infinite loop
208
# this is the reason of this double-pass
209
        unset xmlgetnext_empty_tag property
210
        while xmlgetnext; do
211
        case "${TAG:0:1}" in
212
                ''|'?'|'!')
213
                    continue;;
214
                /)
215
                    property=${property%/*};;
216
                *)
217
                    if test "${TAG: -1}" != '/'; then
218
                        property+=/${TAG%% *}
219
                    fi;;
220
            esac
221

            
222
            if [[ "$property" = /PropertyList@($data_pattern) ]]; then
223
                if test -z "${data[${property/\/PropertyList}]}"; then
224
                    eval "data[${property/\/PropertyList}]=\"${VALUE//\"/\\\"}\""
225
                    data[${property/\/PropertyList}]=$(tr '\n' ' ' <<< ${data[${property/\/PropertyList}]} | sed -r 's/^\s*//;s/\s+/ /g;s/\s*$//')
226
                fi
227
            fi
228

            
229
            # continue parsing (while loop) until everything's found
230
            for col in ${!datatypes[@]}; do
231
                test -z "${data[$col]}" && continue 2
232
            done
233
            break # everything's found
234
        done < $setxml
235

            
236
        if eval "test -z \"$data_test_null\""; then
237
            printf "\nWARNING: no info found, skipping\n"
238
            mkdir -p $temppath/no-data-ac
239
            cp -f $setxml $temppath/no-data-ac/${ac}-${sx}
240
        else
241
            insert_values="'$sx', $id, "
242
            insert_col='file, variantof, '
243
            update_values=''
244
            for col in ${!data[@]}; do
245
                if test ${datatypes[$col]%% *} = 'text'; then
246
                    single_quote="'"
247
                elif [[ ${datatypes[$col]%% *} = 'integer' && "${data[$col]// }" = +([0-9]) ]]; then
248
                    single_quote=""
249
                else
250
                    unset datatypes[$col]
251
                    continue
252
                fi
253
                insert_col+="\`$col\`, "
254
                insert_values+="$single_quote${data[$col]//\'/\'\'}$single_quote, "
255
                if test -n "${data[$col]}"; then
256
                    update_values+="\`$col\` = $single_quote${data[$col]//\'/\'\'}$single_quote, "
257
                fi
258
            done
259
            local flag_new=
260
            local flag_status=
261
            if test $(sqlite_request "select count(file) from setxml where file = '$sx' and variantof = $id") -eq 0; then
262
                flag_new="NEW"
263
            fi
264
            for criteria in FDM model systems cockpit; do
265
                if test ${data[/sim/rating/$criteria]:-0} -ge 4; then
266
                    flag_status+='*'
267
                fi
268
            done
269
            if test -n "$flag_new" -o -n "$flag_status"; then
270
                printf " (${flag_new:+$flag_new }$flag_status)"
271
            fi
272
            printf "\n"
273
            sqlite_request "insert into setxml (${insert_col%,*}, installed) values (${insert_values%,*}, 0)
274
                            on conflict (file, variantof) where file = '$sx' and variantof = $id do
275
                                update set
276
                                    ${update_values%,*}, installed = 0
277
                                where
278
                                    file = '$sx' and variantof = $id"
279
        fi
280

            
281
        sqlite_request "delete from recover_setxml where ac = '$ac' and sx = '$sx'"
282
    done
283
}
284

            
285
function add_record () {
286
    ac_ass_array[$1]="$2"
287
}
288

            
289
function get_record () {
290
    if test -n "$1"; then
291
        echo "${ac_ass_array[$1]}"
292
    else
293
        for k in ${!ac_ass_array[@]}; do
294
            echo $k = ${ac_ass_array[$k]}
295
        done
296
    fi
297
}
298

            
299
function add_aircraft () {
300
    for key in name revision date author; do
301
        test -n "${ac_ass_array[$key]}" # exit if missing data (with the help of "set -e")
302
    done
303
    local new_revision=$(sqlite_request "select revision from recover_aircrafts
304
                                         where name = '${ac_ass_array[name]}'")
305
    if test -z "${new_revision}"; then
306
        sqlite_request "insert into recover_aircrafts (name, revision, date, author, hangar)
307
                            values (
308
                            '${ac_ass_array[name]}',
309
                             ${ac_ass_array[revision]},
310
                             ${ac_ass_array[date]},
311
                            '${ac_ass_array[author]}',
312
                             ${hangar[id]})"
313
    elif test ${new_revision} -lt ${ac_ass_array[revision]//\"}; then
314
        sqlite_request "update recover_aircrafts
315
                        set
316
                            revision =  ${ac_ass_array[revision]},
317
                            date     =  ${ac_ass_array[date]},
318
                            author   = '${ac_ass_array[author]}',
319
                            hangar   =  ${hangar[id]}
320
                        where name = '${ac_ass_array[name]}'"
321
    fi
322
    for key in name revision date author; do
323
        ac_ass_array[$key]=''
324
    done
325
}
326

            
327
function add_setxml_for_aircraft () {
328
    sqlite_request "insert into recover_setxml values ('$1', '${2/%-set.xml}')
329
                    on conflict (ac, sx) where ac = '$1' and sx = '${2/%-set.xml}'
330
                    do nothing"
331
}
332

            
333
function apply_revision () {
334
    for ac in $(sqlite_request "select name from recover_aircrafts"); do
335
        # delete aircrafts that have been deleted from the repo
336
        sqlite_request "delete from setxml
337
                        where (file, variantof) in (
338
                            select file, variantof from setxml
339
                              inner join aircrafts
340
                              where aircrafts.id = setxml.variantof
341
                                and aircrafts.name = '$ac'
342
                                and aircrafts.hangar = ${hangar[id]}
343
                                and setxml.file not in (
344
                                    select sx from recover_setxml where ac = '$ac'
345
                                )
346
                            )"
347

            
348
        # delete aircrafts without setxml found
349
        sqlite_request "delete from recover_aircrafts
350
                        where name not in (select distinct ac from recover_setxml)"
351

            
352
        update_database
353
        if test -d ${hangar[path]}/${ac}/.${hangar[type]} \
354
        && \
355
        case ${hangar[type]} in
356
           svn) test "$(svn info --show-item=url ${hangar[path]}/${ac})" != "${hangar[url]}/${ac}";;
357
           git) test "$(git -C ${hangar[path]}/${ac} config --get remote.origin.url)" != "${hangar[url]}/${ac}.git";;
358
        esac \
359
        || test -d ${hangar[path]}/${ac} -a ! -d ${hangar[path]}/${ac}/.${hangar[type]}; then
360
            echo "INFO: local ${ac} installed out from repo" >&2
361
        fi
362
        sqlite_request "delete from recover_aircrafts where name = '$ac'"
363
    done
364
}
365

            
366
trap trap_break INT
367
trap trap_exit EXIT
368

            
369
stty -echoctl
370

            
371
declare -A hangar
372
data_pattern=$(printf "%s|" ${!datatypes[@]})
373
data_pattern=${data_pattern:0:-1}
374
data_test_null=$(printf '${data[%s]}' ${!datatypes[@]})
375

            
376
if test -e $database; then
377
    cp $database $in_ram_database
378

            
améliore la vérification de ...
Sébastien MARQUE authored on 2021-11-12
379
    sql_cols=$(sqlite_request "pragma table_info(setxml)" \
380
             | cut --delimiter="|" --fields=2,3 --output-delimiter=" " \
381
             | sort \
382
             | md5sum)
383
    datatypes[file]=text
384
    datatypes[variantof]=integer
385
    datatypes[installed]=integer
386
    script_cols=$(for col in ${!datatypes[@]}; do echo "$col ${datatypes["$col"]%% *}"; done \
387
                | sort \
388
                | md5sum)
389
    unset datatypes[file] datatypes[variantof] datatypes[installed]
déplacement depuis dépôt con...
Sébastien MARQUE authored on 2021-05-28
390
    if test "$sql_cols" != "$script_cols"; then
391
        echo "ALERT: datbase version mismatch !"
392
        exit 1
393
    fi
394
    if sqlite_request '.tables' | grep -q 'recover_'; then
395
        hangar[id]=$(sqlite_request "select hangar from recover_aircrafts limit 1")
fix empty recovery
Sébastien MARQUE authored on 2021-09-25
396
        if test -n "${hangar[id]}"; then
397
            echo "recovering from previous saved state"
398
            eval $(sqlite_request "select printf('hangar[name]=%s;hangar[url]=%s;hangar[type]=%s;hangar[source]=%s',
399
                                                  name,           url,           type,           source)
400
                                   from hangars
401
                                   where id = '${hangar[id]}'")
402
            source $(grep -l "^\s*hangar\[name\]=${hangar[name]}\s*$" ${0%*/}.d/*.hangar)
403
            eval "getfromrepo () {$(declare -f getfromrepo | sed '1,2d;$d'); xmlremovecomments;}"
404
            apply_revision
405
            exit
406
        else
407
            sqlite_request 'drop table recover_aircrafts'
408
            sqlite_request 'drop table recover_setxml'
409
        fi
déplacement depuis dépôt con...
Sébastien MARQUE authored on 2021-05-28
410
    fi
411
fi
412

            
413
sqlite_request "create table if not exists hangars (
414
                    id     integer primary key,
415
                    name   text,
416
                    source text,
417
                    type   text,
418
                    url    text,
419
                    path   text,
420
                    active integer)"
421

            
422
sqlite_request 'create unique index if not exists "index_hangars" on hangars (url)'
423

            
424
sqlite_request "create table if not exists aircrafts (
425
                    id       integer primary key,
426
                    name     text,
427
                    revision integer,
428
                    date     integer,
429
                    author   text,
430
                    hangar   integer)"
431

            
432
sqlite_request 'create unique index if not exists "index_aircrafts" on aircrafts (name, hangar)'
433

            
434
sqlite_request "create table if not exists setxml (
435
                    file text,
436
                    variantof integer,
437
                    $(for col in ${!datatypes[@]}; do printf "'%s' %s, " $col "${datatypes[$col]}"; done)
438
                    installed integer)"
439

            
440
sqlite_request 'create unique index if not exists "index_setxml" on setxml (file, variantof)'
441

            
442
for file in $(find ${0%*/}.d -type f -name "*.hangar"); do
443
    unset hangar error_message
444
    unset -f getfromrepo parse_repo_history
445
    declare -A hangar
446
    source $file
447

            
448
    test -n "${hangar[name]}"      \
449
      -a -n "${hangar[source]}"    \
450
      -a -n "${hangar[type]}"      \
451
      -a -n "${hangar[url]}"       \
452
      -a -n "${hangar[active]}" || \
453
        error_message="${error_message:+$error_message, }missing hangar data"
454

            
455
    declare -f getfromrepo > /dev/null || \
456
        error_message="${error_message:+$error_message, }missing getfromrepo function"
457

            
458
    declare -f parse_repo_history > /dev/null || \
459
        error_message="${error_message:+$error_message, }missing parse_repo_history function"
460

            
461
    if test -n "$error_message"; then
462
        echo "file $file isn't a valid hangar ($error_message)"
463
        continue
464
    fi
465

            
466
    sqlite_request "insert into hangars (name, source, type, url, path, active)
467
                    values (
468
                        '${hangar[name]}', '${hangar[source]}', '${hangar[type]}',
469
                        '${hangar[url]}',  '${hangar[path]}',    ${hangar[active]})
470
                    on conflict (url) where url = '${hangar[url]}' do
471
                    update set
472
                        name = '${hangar[name]}',
473
                        path = '${hangar[path]}',
474
                        active = ${hangar[active]}
475
                    where url = '${hangar[url]}'"
476
done
477

            
478
unset hangar
479
unset -f getfromrepo parse_repo_history
480
declare -A hangar ac_ass_array
481
for h_id in $(sqlite_request "select id from hangars where active = 1"); do
482

            
483
    sqlite_request 'create table if not exists recover_aircrafts (
484
                        name     text,
485
                        revision integer,
486
                        date     integer,
487
                        author   text,
488
                        hangar   integer)'
489

            
490
    sqlite_request 'create table if not exists recover_setxml (
491
                        ac text,
492
                        sx text)'
493

            
494
    sqlite_request 'create unique index if not exists "index_recover_setxml" on recover_setxml (ac, sx)'
495

            
496
    eval $(sqlite_request "select printf('hangar[id]=%i;hangar[source]=%s;', id, source)
497
                           from hangars
498
                           where id = '${h_id}'")
499

            
500
    source $(grep -l "^\s*hangar\[source\]=${hangar[source]}\s*$" ${0%*/}.d/*.hangar)
501

            
502
    eval "getfromrepo () {$(declare -f getfromrepo | sed '1,2d;$d'); xmlremovecomments;}"
503

            
504
    echo -e "=${hangar[name]//?/=}=\n ${hangar[name]} \n=${hangar[name]//?/=}="
505

            
506
    latest_revision=$(( $(sqlite_request "select max(revision)
507
                                          from aircrafts inner join hangars
508
                                          where hangars.id = aircrafts.hangar and hangars.name = '${hangar[name]}'") + 1 ))
509

            
510
    parse_repo_history
511

            
512
    if declare -f on_exit > /dev/null; then
513
        on_exit
514
    fi
515
    sqlite_request "drop index 'index_recover_setxml'"
516
    sqlite_request "drop table recover_aircrafts"
517
    sqlite_request "drop table recover_setxml"
518
done
519