#!/bin/bash for tool in sqlite3 getopt md5sum mktemp; do which $tool > /dev/null 2>&1 || { echo missing tool $tool exit 1 } done function create_database () { declare -A groupes if test -z "${groupes[@]}"; then echo "unable to find groupes in $config_file" >&2 exit 1 fi sqlite3 "$in_ram_database" <<< "create table if not exists votes (id integer primary key, nom text)" sqlite3 "$in_ram_database" <<< "create table if not exists url (id integer primary key autoincrement, url text)" sqlite3 "$in_ram_database" <<< "create table if not exists députés (id integer primary key autoincrement, nom text, groupe integer)" sqlite3 "$in_ram_database" <<< "create table if not exists groupes (id integer primary key autoincrement, nom text, nom_court text)" sqlite3 "$in_ram_database" <<< "create table if not exists scrutins (num integer primary key, date text not null, intitulé text non null, adoption boolean, url integer)" sqlite3 "$in_ram_database" <<< "create table if not exists dépouillement (député integer not null, scrutin integer not null, groupe integer not null, vote integer not null)" v_id=0 for v in Pour Contre Abstention Non-votant; do if test -z $(sqlite3 "$in_ram_database" <<< "select nom from votes where id is $v_id"); then sqlite3 "$in_ram_database" <<< "insert into votes values ($v_id, '$v')" else test -z $(sqlite3 "$in_ram_database" <<< "select nom from votes where id is $v_id and nom is '$v'") \ && sqlite3 "$in_ram_database" <<< "update votes set nom = '$v' where id is $v_id)" fi let v_id++ done unset v_id v for g in ${!groupes[@]}; do test -z $(sqlite3 "$in_ram_database" <<< "select id from groupes where nom is '${groupes[$g]}' and nom_court is '$g'") \ && sqlite3 "$in_ram_database" <<< "insert into groupes (nom, nom_court) values ('${groupes[$g]}', '$g')" done unset g groupes test -z $(sqlite3 "$in_ram_database" <<< "select id from url where id = 0") \ && sqlite3 "$in_ram_database" <<< "insert into url values (0, '')" } function update_database () { test "$no_db_update" = $true_flag && return tempfile="/dev/shm/scrutin.$$" progress=0 if test ${first:-0} -lt $last; then echo "récupération des scrutins n°$((${first:-0}+1)) à n°$last dans "$database" (à conserver autant que possible)" >&2 url_database=/dev/shm/url_database : > "$url_database" test $((last % 100)) -ne 0 && last_offset=0 for offset in $(seq $((last - 100)) -100 ${first:-0} ) $last_offset; do wget -qO- "http://www2.assemblee-nationale.fr/scrutins/liste/(offset)/$offset/(legislature)/15/(type)/TOUS/(idDossier)/TOUS" \ | awk ' // { scrutin = gensub(/^.+denom.>([[:digit:]]+).*<.td./,"\\1","1",$0) } /.+dossier<.a/ { a[scrutin] = gensub(/^.+.dossier<.a>.*$/,"\\1","1",$0) } END { for (i in a) print gensub("*","","1",i) "|" a[i] }' >> "$url_database" done sort -u "$url_database" > "${url_database}.sorted" mv -f "${url_database}.sorted" "$url_database" IFS=$'\n' begin=$(date +%s) for scrutin in $(seq $((${first:-0}+1)) $last); do wget -qO- "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/15/(num)/$scrutin" \ | sed -r '0,/< *div class="titre-bandeau-bleu +to-print" *>/d; /< *script +type="text\/javascript" *>/,$d' > $tempfile unset title date adoption url id_url title=$(sed -rn '/

Analyse du scrutin n° '$scrutin'/n; s,^.*

(.+).

,\1,p' $tempfile \ | sed "s/;//g; s/[ \t][ \t]+/ /g; s/^Scrutin public sur *//; s/^l[ae']s* *//") date=$(sed -rn 's,^.*

Analyse du scrutin n° '$scrutin'
(.+)

,\1,p' $tempfile) adoption=$(sed -rn 's,^.*

(.+).

.*$,\1,p' $tempfile) test -n "$title" -a -n "$date" -a -n "$adoption" || { echo "erreur dans la récupération du scrutin $scrutin" exit 1 } grep -q 'e a a' <<< "$adoption" && adoption=1 || adoption=0 url=$(awk -F'|' "/^$scrutin\|/{print \$2}" "$url_database") id_url=$(sqlite3 "$in_ram_database" <<< "select id from url where url is '$url'") if test -z "$id_url"; then sqlite3 "$in_ram_database" <<< "insert into url (url) values ('$url')" id_url=$(sqlite3 "$in_ram_database" <<< "select id from url where url is '$url'") fi sqlite3 "$in_ram_database" <<< "insert into scrutins values ($scrutin, '$date', \"${title//\"}\", $adoption, ${id_url:-0})" for v in $(sqlite3 "$in_ram_database" <<< "select * from votes"); do for g in $(sqlite3 "$in_ram_database" <<< "select id,nom from groupes"); do for d in $(sed -rn '/

'${g#*|}' /,/

/p' $tempfile \ | sed -rn '/

'${v#*|}':/,/<.div>/p' \ | sed 's,,\n,g' \ | sed -rn '/

/d; s,^\s*

  • \s*,,; s, , ,g; s/^\s*//; s/M(me|\.) //; s/ \(.*$//; s,,,; s,,,p'); do d_id=$(sqlite3 "$in_ram_database" <<< "select id from députés where nom is \"$d\" and groupe is ${g%|*}") if test -z "$d_id"; then sqlite3 "$in_ram_database" <<< "insert into députés (nom, groupe) values (\"$d\", ${g%|*})" d_id=$(sqlite3 "$in_ram_database" <<< "select id from députés where nom is \"$d\" and groupe is ${g%|*}") fi sqlite3 "$in_ram_database" <<< "insert into dépouillement values ($d_id, $scrutin, ${g%|*}, ${v%|*})" done done done if test $((($scrutin*100)/$last)) -ne $progress; then progress=$((($scrutin*100)/$last)) if test $(($progress % ${update_progress:-1})) -eq 0; then now=$(date +%s) delta=$(( $now - $begin )) echo $progress%, ETA: $(date +%H:%M:%S -d "$(($delta * ($last - $scrutin) / $scrutin)) seconds") fi fi done rm -f "$url_database" "$tempfile" fi } function write_comparaison () { result="comparaisons $cible avec $groupe_ref${dossier:+ - ${dossier}}" content="/dev/shm/$result/content.xml" id_cols="Scrutin Date Titre Adoption" typevotes=$(sqlite3 "$in_ram_database" <<< "select nom from votes") nb_cols=$(wc -w <<< "$id_cols $typevotes $typevotes") echo "génération du fichier $result" mkdir -p "/dev/shm/$result/META-INF" cat > "/dev/shm/$result/META-INF/manifest.xml" << EOmetainf EOmetainf printf 'application/vnd.oasis.opendocument.spreadsheet' > "/dev/shm/$result/mimetype" echo '' > "$content" cat >> "$content" << EOcontent EOcontent for i in $(seq $nb_cols); do cat >> "$content" << EOcontent EOcontent done cat >> "$content" << EOcontent EOcontent for i in $(seq $(wc -w <<< $typevotes)); do cat >> "$content" << EOcontent EOcontent done echo '' >> "$content" for colonne in $id_cols; do cat >> "$content" << EOcontent $colonne EOcontent done for typevote in $typevotes; do for g in "$cible" $groupe_ref; do cat >> "$content" << EOcontent $typevote - $g EOcontent done done echo '' >> "$content" progress=0 begin=$(date +%s) line=1 for scrutin in $(eval ${seq:-seq $first $last}); do data=$(sqlite3 "$in_ram_database" <<< "select date,intitulé,adoption,url.url from scrutins inner join url on scrutins.url = url.id where num is $scrutin") date=$(cut -d'|' -sf 1 <<< $data) title=$(cut -d'|' -sf 2 <<< $data) adoption=$(cut -d'|' -sf 3 <<< $data) url=$(cut -d'|' -sf 4 <<< $data) test $adoption -eq 1 && adoption='oui' || adoption='non' cat >> "$content" << EOcontent EOcontent if test -n "$url"; then cat >> "$content" << EOcontent $scrutin EOcontent else cat >> "$content" << EOcontent $scrutin EOcontent fi cat >> "$content" << EOcontent $date ${title//\'/'} ${adoption} EOcontent for typevote in 0 1 2 3; do cible_votes=$(sqlite3 "$in_ram_database" <<< "select count(député) from dépouillement where scrutin is $scrutin and vote is $typevote and groupe is $groupe_id ${nom:+ and député is ${nom%|*}}") ref_votes=$(sqlite3 "$in_ram_database" <<< "select count(député) from dépouillement where scrutin is $scrutin and vote is $typevote and groupe is $groupe_ref_id") cat >> "$content" << EOcontent $cible_votes $ref_votes EOcontent done echo '' >> "$content" if test $(( ($line * 100) / ${qty:-$last} )) -ne $progress; then progress=$(( ($line * 100) / ${qty:-$last} )) if test $(( $progress % ${generation_progress:-5} )) -eq 0; then now=$(date +%s) delta=$(( $now - $begin )) echo $progress%, ETA: $(date +%H:%M:%S -d "$(( $delta * (${qty:-$last} - $line) / $line )) seconds") fi fi let line++ done echo cat >> "$content" << EOcontent EOcontent ( cd "/dev/shm/$result" && zip -r ../"$result" * > /dev/null 2>&1 && cd .. && rm -fr "$result" ) mv -f "/dev/shm/$result.zip" "$result.ods" echo "$result.ods" } function save_database () { if test -r "$database" && md5sum $in_ram_database | sed "s,$in_ram_database,$database," | md5sum --status -c -; then rm -f $in_ram_database elif test -w "$database"; then mv -f $in_ram_database "$database" elif ! test -e "$database"; then mv $in_ram_database "$database" else rm -f $in_ram_database fi } trap save_database EXIT true_flag=$(mktemp --dry-run XXXXX) OPTS=$( getopt -l no-db-update,\ db-update-only,\ cible:,\ ref:,\ député:,\ premier-scrutin:,\ dernier-scrutin:,\ période:,\ liste-dossiers,\ liste-députés,\ dossiers,\ dossier:,\ conf:,\ database:,\ progrès-génération:\ progrès-update: \ -- "$@" ) eval set --$OPTS while [[ $# -gt 0 ]]; do case "$1" in "--no-db-update") no_db_update=$true_flag;; "--db-update-only") db_update_only=$true_flag;; "--cible") groupe="${2^^}" shift;; "--ref") groupes_ref="${2^^}" shift;; "--député") depute=$true_flag nom="$2" shift;; "--premier-scrutin") no_db_update=$true_flag first="$2" shift;; "--dernier-scrutin") no_db_update=$true_flag last="$2" shift;; "--période") periode=$true_flag no_db_update=$true_flag periode_value="$2" shift;; "--liste-députés-du-groupe") liste_deputes=$true_flag liste_deputes_value="${2^^}" shift;; "--liste-députés") liste_deputes=$true_flag;; "--liste-dossiers") liste_dossiers=$true_flag;; "--dossier") dossier=$true_flag dossier_value="$2" shift;; "--dossiers") dossier=$true_flag;; "--conf") test -r "$2" || { echo "config introuvable $2" >&2 options_error=$true_flag } config_file="$2" shift;; "--database") test -r "$2" && file -b "$2" | grep -q '^SQLite 3.x database' || { echo "erreur sur option database: fichier '$2' introuvable ou pas une base SQLite 3" >&2 options_error=$true_flag } database="$2" shift;; "--progrès-génération") generation_progress="$2" shift;; "--progrès-update") update_progress="$2" shift;; esac shift done test "$options_error" = $true_flag && exit 1 test -z "$database" && database="${0}.db" if test -n "$config_file"; then source "$config_file" else config_file="${0}.conf" if test -r "$config_file"; then source "$config_file" fi fi groupe=${groupe:-LREM} groupe_ref=${groupe_ref:-GDR} cible=$groupe in_ram_database=$(mktemp --dry-run /dev/shm/XXXXXXXXXXXX) if test -r "$database"; then cp "$database" "$in_ram_database" else create_database fi for g in groupe groupe_ref; do eval "${g}_id=$(sqlite3 "$in_ram_database" <<< "select id from groupes where nom_court is '${!g}'")" if eval "test -z \$${g}_id"; then echo "groupe ${!g} inconnu" >&2 exit 1 fi done if test "$periode" = $true_flag; then first=$(sqlite3 "$in_ram_database" <<< "select num from scrutins where date like '% du ${periode_value%:*}' order by num asc" | head -1) last=$(sqlite3 "$in_ram_database" <<< "select num from scrutins where date like '% du ${periode_value#*:}' order by num asc" | tail -1) test -z "$first" && echo "date de début inconnue: ${periode_value#*:}" >&2 && rm -f $in_ram_database && exit 1 test -z "$last" && echo "date de fin inconnue: ${periode_value%:*}" >&2 && rm -f $in_ram_database && exit 1 else test -z "$last" && last=$(wget -qO- 'http://www2.assemblee-nationale.fr/scrutins/liste/(legislature)/15/(type)/TOUS/(idDossier)/TOUS' \ | sed -rn 's,^.*(.+).*$,\1,p' \ | head -1) test -z "$first" && first=$(sqlite3 "$in_ram_database" <<< "select count(num) from scrutins") fi if test "$liste_dossiers" = $true_flag; then sqlite3 "$in_ram_database" <<< "select printf('%s - %s', id, url) from url" | sed 's,https*://.*/dossiers/,,; s/_/ /g; s/.asp$//' exit fi if test "$db_update_only" = $true_flag; then unset first last update_database exit fi if test "$liste_deputes" = $true_flag; then if test -n "$liste_deputes_value"; then sqlite3 "$in_ram_database" <<< "select printf('%s - %s', députés.nom, groupes.nom_court) from députés inner join groupes on groupes.id = députés.groupe where groupes.nom_court is '$liste_deputes_value'" else sqlite3 "$in_ram_database" <<< "select printf('%s - %s', députés.nom, groupes.nom_court) from députés inner join groupes on groupes.id = députés.groupe order by groupes.nom_court asc" fi exit fi if test "$depute" = $true_flag; then if test -n "$nom"; then match=$(sqlite3 "$in_ram_database" <<< "select count(députés.id) from députés inner join groupes on groupes.id = députés.groupe where députés.nom like '%$nom%' and groupes.nom_court is '$groupe' collate nocase") if test $match -ne 1; then if test $match -eq 0; then echo "pas de député correspondant dans le groupe $groupe" else echo "plusieurs députés correspondent:" sqlite3 "$in_ram_database" <<< "select députés.nom from députés inner join groupes on groupes.id = députés.groupe where députés.nom like '%$nom%' and groupes.nom_court is '$groupe' collate nocase" fi exit 1 else nom=$(sqlite3 "$in_ram_database" <<< "select députés.id,députés.nom from députés inner join groupes on groupes.id = députés.groupe where députés.nom like '%$nom%' and groupes.nom_court is '$groupe' collate nocase") cible="${nom#*|} ($groupe)" fi fi fi if test "$dossier" = $true_flag; then if test -z "$dossier_value"; then IFS_=$IFS IFS=$'\n' select dossier in $(sqlite3 "$in_ram_database" <<< "select url from url" | sed 's,^.*/dossiers/,,; s/_/ /g; s/.asp$//'); do if test -n "$dossier"; then seq="sqlite3 \"$in_ram_database\" <<< \"select num from scrutins inner join url on url.id = scrutins.url where url.url like '%/dossiers/${dossier// /_}%' order by num asc\"" qty=$(sqlite3 "$in_ram_database" <<< "select count(num) from scrutins inner join url on url.id = scrutins.url where url.url like '%/dossiers/${dossier// /_}%' order by num asc") break fi done IFS=$IFS_ else seq="sqlite3 \"$in_ram_database\" <<< \"select num from scrutins inner join url on url.id = scrutins.url where url.id is $dossier_value order by num asc\"" qty=$(sqlite3 "$in_ram_database" <<< "select count(num) from scrutins inner join url on url.id = scrutins.url where url.id is $dossier_value order by num asc") dossier=$(sqlite3 "$in_ram_database" <<< "select url from url where id is $dossier_value" | sed 's,^.*/dossiers/,,; s/_/ /g; s/.asp$//') fi fi update_database write_comparaison