Make postgres database configurable (#335)

avancement_cumul.sh

@@ -2,7 +2,10 @@
 
 set -e
 
-psql -d cadastre -U cadastre -c "SELECT  source,etape, date_debut,date_fin,dept,cadastre_com,nom_com,nombre_adresses FROM batch ORDER BY id_batch DESC LIMIT 10;"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $SCRIPT_DIR/config
+
+$pgsql_CADASTRE -c "SELECT  source,etape, date_debut,date_fin,dept,cadastre_com,nom_com,nombre_adresses FROM batch ORDER BY id_batch DESC LIMIT 10;"
 ps -eaf|grep 'bano'
 ps -eaf|grep cumul
 

bano/db.py

@@ -3,7 +3,7 @@
 import psycopg2
 import psycopg2.extras
 
-bano = psycopg2.connect(os.environ.get("BANO_PG", "dbname='cadastre' user='cadastre'"))
-bano_cache = psycopg2.connect(os.environ.get("BANO_PG_CACHE", "dbname='osm' user='cadastre'"))
+bano = psycopg2.connect(os.environ.get("PG_CADASTRE", "dbname='cadastre' user='cadastre'"))
+bano_cache = psycopg2.connect(os.environ.get("PG_OSM", "dbname='osm' user='cadastre'"))
 bano_cache.autocommit = True
 psycopg2.extras.register_hstore(bano_cache)

bano/export.py

@@ -146,7 +146,7 @@ def save_as_ttl(self):
 
 
     def save_as_shp(self):
-        subprocess.run(['ogr2ogr', '-f',"ESRI Shapefile", '-lco', 'ENCODING=UTF-8', '-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326', '-overwrite', self.get_sas_full_filename('shp'), 'PG:dbname=cadastre user=cadastre', '-sql', f'{self.csv_query}'])
+        subprocess.run(['ogr2ogr', '-f',"ESRI Shapefile", '-lco', 'ENCODING=UTF-8', '-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326', '-overwrite', self.get_sas_full_filename('shp'), 'PG:' + os.environ['PG_CADASTRE'], '-sql', f'{self.csv_query}'])
 
     def save_as_json(self):
         with open(self.get_sas_full_filename('json'),'w') as jsonfile:

config

@@ -16,3 +16,20 @@ export EXPORT_WEB_DIR=${EXPORT_WEB_DIR:-/data/work/bano.openstreetmap.fr/www/web
 export TMP_DIR=${TMP_DIR:-/data/tmp}
 export SCHEMA_CIBLE=public
 export LANG=${LANG:-fr_FR.UTF-8}
+
+
+POSTGRES_USER=${POSTGRES_USER:-cadastre}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
+POSTGRES_HOST=${POSTGRES_HOST:-localhost}
+POSTGRES_PORT=${POSTGRES_PORT:-5432}
+
+POSTGRES_DB_OSM=${POSTGRES_DB:-osm}
+POSTGRES_DB_CADASTRE=${POSTGRES_DB:-cadastre}
+
+export PGCON_OSM="$POSTGRES_USER${POSTGRES_PASSWORD:+:$POSTGRES_PASSWORD}@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB_OSM"
+export pgsql_OSM="psql postgresql://${PGCON_OSM} -v ON_ERROR_STOP=1"
+export PG_OSM="host='$POSTGRES_HOST' port=$POSTGRES_PORT dbname='$POSTGRES_DB_OSM' user='$POSTGRES_USER' ${POSTGRES_PASSWORD:+password='$POSTGRES_PASSWORD'}"
+
+export PGCON_CADASTRE="$POSTGRES_USER${POSTGRES_PASSWORD:+:$POSTGRES_PASSWORD}@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB_CADASTRE"
+export pgsql_CADASTRE="psql postgresql://${PGCON_CADASTRE} -v ON_ERROR_STOP=1"
+export PG_CADASTRE="host='$POSTGRES_HOST' port=$POSTGRES_PORT dbname='$POSTGRES_DB_CADASTRE' user='$POSTGRES_USER' ${POSTGRES_PASSWORD:+password='$POSTGRES_PASSWORD'}"

copy_table_from_osm_to_cadastre.sh

@@ -8,7 +8,7 @@ source $SCRIPT_DIR/config
 set -e
 
 cd $TMP_DIR
-pg_dump -t ${1} -U cadastre osm > ${1}_dump.sql
-psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS ${1} CASCADE;"
-psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f ${1}_dump.sql
+pg_dump -t ${1} "postgresql://${PGCON_OSM}" > ${1}_dump.sql
+$pgsql_CADASTRE -c "DROP TABLE IF EXISTS ${1} CASCADE;"
+$pgsql_CADASTRE -f ${1}_dump.sql
 rm -f ${1}_dump.sql

create_base.sh

@@ -2,6 +2,8 @@
 
 set -e
 
-createuser -s -d cadastre
-createdb -E UTF-8 -T template0 -O cadastre cadastre
-createdb -E UTF-8 -T template0 -O cadastre osm
+source config
+
+createuser -s -d $POSTGRES_USER
+createdb -E UTF-8 -T template0 -O $POSTGRES_USER $POSTGRES_DB_CADASTRE
+createdb -E UTF-8 -T template0 -O $POSTGRES_USER $POSTGRES_DB_OSM

cron_bano.sh

@@ -48,14 +48,14 @@ fi
 ./copy_table_from_osm_to_cadastre.sh planet_osm_postal_code
 ./copy_table_from_osm_to_cadastre.sh infos_communes
 
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_polygones_communes.sql
+$pgsql_OSM -f sql/create_table_polygones_communes.sql
 ./copy_table_from_osm_to_cadastre.sh polygones_insee
 ./copy_table_from_osm_to_cadastre.sh polygones_insee_geo
 
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_polygones_postaux.sql
+$pgsql_OSM -f sql/create_table_polygones_postaux.sql
 ./copy_table_from_osm_to_cadastre.sh polygones_postaux
 ./copy_table_from_osm_to_cadastre.sh ban_odbl
-psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f sql/post_copie_ban_odbl.sql
+$pgsql_CADASTRE -f sql/post_copie_ban_odbl.sql
 
 # exports
 cat deplist.txt | parallel -j 4 bano export {1}
@@ -65,7 +65,7 @@ cat deplist.txt | parallel -j 4 bano publish {1}
 bano publish_aggregate
 
 # ménage PostgreSQL
-psql -d cadastre -U cadastre -c "VACUUM cumul_adresses;"
-psql -d cadastre -U cadastre -c "VACUUM cumul_voies;"
-psql -d cadastre -U cadastre -c "VACUUM cumul_places;"
-psql -d cadastre -U cadastre -c "GRANT SELECT ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC";
+$pgsql_CADASTRE -c "VACUUM cumul_adresses;"
+$pgsql_CADASTRE -c "VACUUM cumul_voies;"
+$pgsql_CADASTRE -c "VACUUM cumul_places;"
+$pgsql_CADASTRE -c "GRANT SELECT ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC";

init_base.sh

@@ -2,7 +2,9 @@
 
 set -e
 
-psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f bano/sql/create_base.sql
-psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_base_bano.sql
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f bano/sql/create_base.sql
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_base_osm.sql
+source config
+
+$pgsql_CADASTRE -f bano/sql/create_base.sql
+$pgsql_CADASTRE -f sql/create_table_base_bano.sql
+$pgsql_OSM -f bano/sql/create_base.sql
+$pgsql_OSM -f sql/create_table_base_osm.sql

load_COG.sh

@@ -16,12 +16,12 @@ CSV_ARRONDISSEMENT=arrondissement2021.csv
 CSV_DEPARTEMENT=departement2021.csv
 CSV_REGION=region2021.csv
 
-for d in osm cadastre
+for psql in "$pgsql_CADASTRE" "$pgsql_OSM"
 do
-  psql -d ${d} -U cadastre -f $BANO_DIR/sql/create_table_cog.sql
-  cat ${CSV_COMMUNE}|       psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_commune;       COPY cog_commune FROM stdin WITH csv header delimiter ',';"
-  cat ${CSV_CANTON}|        psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_canton;        COPY cog_canton FROM stdin WITH csv header delimiter ',';"
-  cat ${CSV_ARRONDISSEMENT}|psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_arrondissement;COPY cog_arrondissement FROM stdin WITH csv header delimiter ',';"
-  cat ${CSV_DEPARTEMENT}|   psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_departement;   COPY cog_departement FROM stdin WITH csv header delimiter ',';"
-  cat ${CSV_REGION}|        psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_region;        COPY cog_region FROM stdin WITH csv header delimiter ',';"
+  $psql -f $BANO_DIR/sql/create_table_cog.sql
+  cat "${CSV_COMMUNE}"        | $psql -c "TRUNCATE TABLE cog_commune;       COPY cog_commune FROM stdin WITH csv header delimiter ',';"
+  cat "${CSV_CANTON}"         | $psql -c "TRUNCATE TABLE cog_canton;        COPY cog_canton FROM stdin WITH csv header delimiter ',';"
+  cat "${CSV_ARRONDISSEMENT}" | $psql -c "TRUNCATE TABLE cog_arrondissement;COPY cog_arrondissement FROM stdin WITH csv header delimiter ',';"
+  cat "${CSV_DEPARTEMENT}"    | $psql -c "TRUNCATE TABLE cog_departement;   COPY cog_departement FROM stdin WITH csv header delimiter ',';"
+  cat "${CSV_REGION}"         | $psql -c "TRUNCATE TABLE cog_region;        COPY cog_region FROM stdin WITH csv header delimiter ',';"
 done

load_code_cadastre.sh

@@ -2,5 +2,7 @@
 
 set -e
 
+source config
+
 python load_code_cadastre.py
-psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f bano/sql/load_code_cadastre.sql
+$pgsql_CADASTRE -f bano/sql/load_code_cadastre.sql

load_codes_postaux.sh

@@ -8,5 +8,5 @@ mkdir -p codes_postaux
 cd codes_postaux
 rm -f *csv*
 wget "https://datanova.legroupe.laposte.fr/explore/dataset/laposte_hexasmal/download/?format=csv&timezone=Europe/Berlin&use_labels_for_header=true&csv_separator=%3B" -O codes_postaux.csv
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/create_and_load_codes_postaux.sql
-psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/create_and_load_codes_postaux.sql
+$pgsql_OSM -f $BANO_DIR/sql/create_and_load_codes_postaux.sql
+$pgsql_CADASTRE -f $BANO_DIR/sql/create_and_load_codes_postaux.sql

load_fantoir.sh

@@ -10,5 +10,5 @@ rm -rf FANTOIR* && rm -f *.zip
 wget https://data.economie.gouv.fr/api/datasets/1.0/fichier-fantoir-des-voies-et-lieux-dits/attachments/fichier_national_fantoir_situation_janvier_2023_zip/ -O fantoir.zip
 unzip fantoir.zip
 mv FANTOIR* fantoir.txt
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/load_fantoir.sql
-psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/load_fantoir.sql
+$pgsql_OSM -f $BANO_DIR/sql/load_fantoir.sql
+$pgsql_CADASTRE -f $BANO_DIR/sql/load_fantoir.sql

load_osm_france_db.sh

@@ -22,9 +22,9 @@ mkdir -p $DOWNLOAD_DIR
 cd $DOWNLOAD_DIR
 wget -NS $PBF_URL
 imposm import -mapping $BANO_DIR/bano.yml -read $DOWNLOAD_DIR/$PBF_FILE -overwritecache -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR
-imposm import -mapping $BANO_DIR/bano.yml -write -connection 'postgis://cadastre@localhost/osm'?prefix=NONE -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR
+imposm import -mapping $BANO_DIR/bano.yml -write -connection "postgis://$PGCON_OSM"?prefix=NONE -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR
 
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/finalisation.sql
+$pgsql_OSM -f $BANO_DIR/sql/finalisation.sql
 
 cp $DOWNLOAD_DIR/last.state.txt $DOWNLOAD_DIR/state.txt
 rm ${lockfile}

stats.sh

@@ -2,4 +2,6 @@
 
 set -e
 
-psql -d cadastre -v ON_ERROR_STOP=1 -f stats.sql -v dept=$1
+source config
+
+$pgsql_CADASTRE -f stats.sql -v dept=$1

update_bases_adresses_locales.sh

@@ -12,7 +12,7 @@ do
     echo 'Département '$dep
 	wget -NS https://adresse.data.gouv.fr/data/adresses-locales/latest/csv/adresses-locales-$dep.csv.gz
     quoted_dep=\'$dep'%'\'
-    psql -d osm -U cadastre -c "DELETE FROM bal_locales WHERE commune_code LIKE $quoted_dep;"
-    gunzip -c $BAL_CACHE_DIR/adresses-locales-$dep.csv.gz|psql -d osm -U cadastre -c "\copy bal_locales from stdin with csv header delimiter ';'"
+    $pgsql_OSM -c "DELETE FROM bal_locales WHERE commune_code LIKE $quoted_dep;"
+    gunzip -c $BAL_CACHE_DIR/adresses-locales-$dep.csv.gz| $pgsql_OSM -c "\copy bal_locales from stdin with csv header delimiter ';'"
     sleep 1
 done

update_cadastre_adresses.sh

@@ -10,7 +10,7 @@ do
     echo 'Département '$dep
 	wget -NS https://adresse.data.gouv.fr/data/adresses-cadastre/latest/csv/adresses-cadastre-$dep.csv.gz
     quoted_dep=\'$dep'%'\'
-    psql -d osm -U cadastre -c "DELETE FROM bal_cadastre WHERE commune_code LIKE $quoted_dep;"
-    gunzip -c $CADASTRE_CACHE_DIR/adresses-cadastre-$dep.csv.gz|psql -d osm -U cadastre -c "\copy bal_cadastre from stdin with csv header delimiter ';'"
+    $pgsql_OSM -c "DELETE FROM bal_cadastre WHERE commune_code LIKE $quoted_dep;"
+    gunzip -c $CADASTRE_CACHE_DIR/adresses-cadastre-$dep.csv.gz| $pgsql_OSM -c "\copy bal_cadastre from stdin with csv header delimiter ';'"
     sleep 1
 done

update_cadastre_batiments.sh

@@ -17,8 +17,8 @@ do
 	unzip $CADASTRE_CACHE_DIR/cadastre-$dep-batiments-shp.zip
     # reconstruction du shapefile avec ogr2ogr car corrompu pour shp2pgsql
     ogr2ogr -overwrite -f 'ESRI Shapefile' batiments_ok.shp batiments.shp
-    shp2pgsql -s 2154:4326 -g geometrie -W LATIN1 batiments_ok.shp public.tmp_batiments$dep | psql -d cadastre -q
-    psql -d cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/replace_batiments.sql -v schema_cible=$SCHEMA_CIBLE -v dept=$dep
+    shp2pgsql -s 2154:4326 -g geometrie -W LATIN1 batiments_ok.shp public.tmp_batiments$dep | $pgsql_CADASTRE -q
+    $pgsql_CADASTRE -f $BANO_DIR/sql/replace_batiments.sql -v schema_cible=$SCHEMA_CIBLE -v dept=$dep
 	zip -mT $CADASTRE_CACHE_DIR/cadastre-$dep-batiments-shp.zip batiments.*
     sleep 1
     cd ..

update_cadastre_lieux-dits.sh

@@ -25,7 +25,7 @@ do
         gzip -dfk ${ZIPFILE}
         mv ${JSONFILE} lieux_dits.json
         # ls -al
-        psql -d osm -U cadastre -c "DELETE FROM lieux_dits WHERE insee_com LIKE '${DEPT}%';"
+        $pgsql_OSM -c "DELETE FROM lieux_dits WHERE insee_com LIKE '${DEPT}%';"
         ogr2ogr  -append -nln lieux_dits -fieldmap 1,0,2,3 -f PostgreSQL PG:'user=cadastre dbname=osm' lieux_dits.json
         rm lieux_dits.json
     else

update_table_infos_communes.sh

@@ -5,4 +5,4 @@ set -e
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd $SCRIPT_DIR
 
-psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f ./sql/update_table_infos_communes.sql
+$pgsql_OSM -f ./sql/update_table_infos_communes.sql

utils/load_csv_export.sh

@@ -2,7 +2,10 @@
 
 set -e
 
-psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS export_csv CASCADE;
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $SCRIPT_DIR/config
+
+$pgsql_CADASTRE -c "DROP TABLE IF EXISTS export_csv CASCADE;
                                  CREATE TABLE export_csv(
                                  id text,
                                  numero text,
@@ -15,5 +18,5 @@ psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS export_csv CASCADE;
 
 for dep in {01..19} 2A 2B {21..95} {971..974} 976
 do
-  cat /data/sas_web/bano-${dep}.csv|psql -d cadastre -U cadastre -c "COPY export_csv FROM STDIN WITH CSV"
+  cat /data/sas_web/bano-${dep}.csv| $pgsql_CADASTRE -c "COPY export_csv FROM STDIN WITH CSV"
 done