diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7d2e8d9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +geonode/uploaded +geonode/static_root +geonode/static/.components +geonode/static/node_modules +docs +.coverage +.celerybeat-* + +scripts/spcgeonode/_volume_* diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..d83a7bf --- /dev/null +++ b/.env.sample @@ -0,0 +1,254 @@ +COMPOSE_PROJECT_NAME=geonode +# See https://github.com/containers/podman/issues/13889 +# DOCKER_BUILDKIT=0 +DOCKER_ENV=production +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true + +# LANGUAGE_CODE=it-it +# LANGUAGES=(('en-us','English'),('it-it','Italiano')) + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=geonode + +# ################# +# backend +# ################# +POSTGRES_USER=postgres +POSTGRES_PASSWORD={pgpwd} +GEONODE_DATABASE=geonode +GEONODE_DATABASE_USER=geonode +GEONODE_DATABASE_PASSWORD={dbpwd} +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_USER=geonode_data +GEONODE_GEODATABASE_PASSWORD={geodbpwd} +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=db +DATABASE_PORT=5432 +DATABASE_URL=postgis://geonode:{dbpwd}@db:5432/geonode +GEODATABASE_URL=postgis://geonode_data:{geodbpwd}@db:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=amqp://guest:guest@rabbitmq:5672/ +CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler +ASYNC_SIGNALS=True + +SITEURL={siteurl}/ + +ALLOWED_HOSTS="['django', '{hostname}']" + +# Data Uploader +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False + +# ################# +# nginx +# HTTPD Server +# ################# +GEONODE_LB_HOST_IP=django +GEONODE_LB_PORT=8000 +NGINX_BASE_URL={siteurl} + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST={http_host} +HTTPS_HOST={https_host} + +HTTP_PORT=80 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE={letsencrypt_mode} +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# ################# +# geoserver +# ################# +GEOSERVER_LB_HOST_IP=geoserver +GEOSERVER_LB_PORT=8080 +GEOSERVER_WEB_UI_LOCATION={siteurl}/geoserver/ +GEOSERVER_PUBLIC_LOCATION={siteurl}/geoserver/ +GEOSERVER_LOCATION=http://${GEOSERVER_LB_HOST_IP}:${GEOSERVER_LB_PORT}/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD={geoserverpwd} + +OGC_REQUEST_TIMEOUT=30 +OGC_REQUEST_MAX_RETRIES=1 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS=-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL={siteurl}/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine + +# ################# +# Security +# ################# +# Admin Settings +# +# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time +# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, +# otherwise the invoke updateadmin task will be skipped and the current password already stored +# in DB will honored. + +ADMIN_USERNAME=admin +ADMIN_PASSWORD={geonodepwd} +ADMIN_EMAIL={email} + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='{email}' # eg Company + +# Session/Access Control +LOCKDOWN_GEONODE=False +X_FRAME_OPTIONS="SAMEORIGIN" +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_VIEW_PERMISSION=True +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True + +CORS_ALLOW_ALL_ORIGINS=True +GEOSERVER_CORS_ENABLED=True +GEOSERVER_CORS_ALLOWED_ORIGINS=* +GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS +GEOSERVER_CORS_ALLOWED_HEADERS=* + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_EMAIL_REQUIRED=True +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False +ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False +ACCOUNT_AUTHENTICATION_METHOD=username_email +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID={clientid} +OAUTH2_CLIENT_SECRET={clientsecret} + +# GeoNode APIs +API_LOCKDOWN=False +TASTYPIE_APIKEY= + +# ################# +# Production and +# Monitoring +# ################# +DEBUG={debug} + +SECRET_KEY='{secret_key}' + +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +ASSETS_ROOT=/mnt/volumes/statics/assets/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db + +CACHE_BUSTING_STATIC_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_LOCATION=memcached:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 +# +# Options for memcached binary, e.g. -vvv to log all requests and cache hits +# +MEMCACHED_OPTIONS= + +MAX_DOCUMENT_SIZE=200 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=False +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# Advanced Workflow +RESOURCE_PUBLISHING=False +ADMIN_MODERATE_UPLOADS=False + +# LDAP +LDAP_ENABLED=False +LDAP_SERVER_URL=ldap:// +LDAP_BIND_DN=uid=ldapinfo,cn=users,dc=ad,dc=example,dc=org +LDAP_BIND_PASSWORD= +LDAP_USER_SEARCH_DN=dc=ad,dc=example,dc=org +LDAP_USER_SEARCH_FILTERSTR=(&(uid=%(user)s)(objectClass=person)) +LDAP_GROUP_SEARCH_DN=cn=groups,dc=ad,dc=example,dc=org +LDAP_GROUP_SEARCH_FILTERSTR=(|(cn=abt1)(cn=abt2)(cn=abt3)(cn=abt4)(cn=abt5)(cn=abt6)) +LDAP_GROUP_PROFILE_MEMBER_ATTR=uniqueMember + +# CELERY + +# expressed in KB +# CELERY__MAX_MEMORY_PER_CHILD="200000" +# ## +# Note right autoscale value must coincide with worker concurrency value +# CELERY__AUTOSCALE_VALUES="15,10" +# CELERY__WORKER_CONCURRENCY="10" +# ## +# CELERY__OPTS="--without-gossip --without-mingle -Ofair -B -E" +# CELERY__BEAT_SCHEDULE="/mnt/volumes/statics/celerybeat-schedule" +# CELERY__LOG_LEVEL="INFO" +# CELERY__LOG_FILE="/var/log/celery.log" +# CELERY__WORKER_NAME="worker1@%h" + +# PostgreSQL +POSTGRESQL_MAX_CONNECTIONS=200 + +# Common containers restart policy +RESTART_POLICY_CONDITION="on-failure" +RESTART_POLICY_DELAY="5s" +RESTART_POLICY_MAX_ATTEMPTS="3" +RESTART_POLICY_WINDOW=120s + +DEFAULT_MAX_UPLOAD_SIZE=5368709120 +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 + +# FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration \ No newline at end of file diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..598014b --- /dev/null +++ b/.env.template @@ -0,0 +1,55 @@ +# ================================================================= +# GeoNode Keycloak Integration Template +# ================================================================= +# Copia questo file in .env e compila le variabili necessarie + +# ================================================================= +# KEYCLOAK CONFIGURATION +# ================================================================= +# URL del server Keycloak (senza trailing slash) +KEYCLOAK_SERVER_URL=https://your-keycloak-server.com + +# Nome del realm Keycloak +KEYCLOAK_REALM=your-realm + +# Client ID registrato in Keycloak +KEYCLOAK_CLIENT_ID=your-client-id + +# Client Secret (da generare in Keycloak) +KEYCLOAK_CLIENT_SECRET=your-client-secret + +# Audience per la validazione del token +KEYCLOAK_AUDIENCE=account + +# Verifica SSL (True in produzione, False solo per test) +KEYCLOAK_VERIFY_SSL=True + +# ================================================================= +# OIDC PROVIDER CONFIGURATION +# ================================================================= +# Abilita/disabilita il provider OIDC +SOCIALACCOUNT_OIDC_PROVIDER_ENABLED=True + +# Nome del provider OIDC (non modificare) +SOCIALACCOUNT_OIDC_PROVIDER=geonode_openid_connect + +# Nome del provider sociale (non modificare) +SOCIALACCOUNT_PROVIDER=keycloak + +# Creazione automatica utenti al primo login +SOCIALACCOUNT_AUTO_SIGNUP=True + +# Disabilita il signup locale se usi solo Keycloak +SOCIALACCOUNT_WITH_GEONODE_LOCAL_SINGUP=False + +# ================================================================= +# OIDC ENDPOINTS (generati automaticamente, non modificare) +# ================================================================= +OIDC_AUTHORIZATION_URL=${KEYCLOAK_SERVER_URL}/auth/realms/${KEYCLOAK_REALM}/protocol/openid-connect/auth +OIDC_ACCESS_TOKEN_URL=${KEYCLOAK_SERVER_URL}/auth/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token +OIDC_USER_INFO_URL=${KEYCLOAK_SERVER_URL}/auth/realms/${KEYCLOAK_REALM}/protocol/openid-connect/userinfo + +# ================================================================= +# ALTRI SETTINGS GEONODE (mantieni quelli esistenti) +# ================================================================= +DEBUG=False \ No newline at end of file diff --git a/.env_dev b/.env_dev new file mode 100644 index 0000000..34f85b4 --- /dev/null +++ b/.env_dev @@ -0,0 +1,225 @@ +COMPOSE_PROJECT_NAME=geonode +# See https://github.com/containers/podman/issues/13889 +# DOCKER_BUILDKIT=0 +DOCKER_ENV=production +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true + +# LANGUAGE_CODE=it-it +# LANGUAGES=(('en-us','English'),('it-it','Italiano')) + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=geonode + +# ################# +# backend +# ################# +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +GEONODE_DATABASE=geonode +GEONODE_DATABASE_USER=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_USER=geonode +GEONODE_GEODATABASE_PASSWORD=geonode +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=localhost +DATABASE_PORT=5432 +DATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode +GEODATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=amqp://admin:admin@localhost:5672// +CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler +ASYNC_SIGNALS=False + +SITEURL=http://localhost:8000/ + +ALLOWED_HOSTS="['django', '*']" + +# Data Uploader +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False + +# ################# +# nginx +# HTTPD Server +# ################# +GEONODE_LB_HOST_IP=django +GEONODE_LB_PORT=8000 +GEOSERVER_LB_HOST_IP=geoserver +GEOSERVER_LB_PORT=8080 +NGINX_BASE_URL=http://localhost + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST=localhost +HTTPS_HOST= + +HTTP_PORT=8000 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE=disabled +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# ################# +# geoserver +# ################# +GEOSERVER_WEB_UI_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_PUBLIC_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD=geoserver + +OGC_REQUEST_TIMEOUT=30 +OGC_REQUEST_MAX_RETRIES=1 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost:8080/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' + +# ################# +# Security +# ################# +# Admin Settings +# +# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time +# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, +# otherwise the invoke updateadmin task will be skipped and the current password already stored +# in DB will honored. + +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +ADMIN_EMAIL=admin@localhost + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='GeoNode ' + +# Session/Access Control +LOCKDOWN_GEONODE=False +X_FRAME_OPTIONS="SAMEORIGIN" +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_VIEW_PERMISSION=True +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True + +CORS_ALLOW_ALL_ORIGINS=True +GEOSERVER_CORS_ENABLED=True +GEOSERVER_CORS_ALLOWED_ORIGINS=* +GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS +GEOSERVER_CORS_ALLOWED_HEADERS=* + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_EMAIL_REQUIRED=True +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False +ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False +ACCOUNT_AUTHENTICATION_METHOD=username_email +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV +OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 + +# GeoNode APIs +API_LOCKDOWN=False +TASTYPIE_APIKEY= + +# ################# +# Production and +# Monitoring +# ################# +DEBUG=True + +SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' + +# STATIC_ROOT=/mnt/volumes/statics/static/ +# MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +# GEOIP_PATH=/mnt/volumes/statics/geoip.db + +CACHE_BUSTING_STATIC_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_LOCATION=127.0.0.1:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 +# +# Options for memcached binary, e.g. -vvv to log all requests and cache hits +# +MEMCACHED_OPTIONS= + +MAX_DOCUMENT_SIZE=200 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=False +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# Advanced Workflow +RESOURCE_PUBLISHING=False +ADMIN_MODERATE_UPLOADS=False + +# PostgreSQL +POSTGRESQL_MAX_CONNECTIONS=200 + +# Common containers restart policy +RESTART_POLICY_CONDITION="on-failure" +RESTART_POLICY_DELAY="5s" +RESTART_POLICY_MAX_ATTEMPTS="3" +RESTART_POLICY_WINDOW=120s + +DEFAULT_MAX_UPLOAD_SIZE=5368709120 +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 diff --git a/.env_local b/.env_local new file mode 100644 index 0000000..3045a3e --- /dev/null +++ b/.env_local @@ -0,0 +1,225 @@ +COMPOSE_PROJECT_NAME=geonode +# See https://github.com/containers/podman/issues/13889 +# DOCKER_BUILDKIT=0 +DOCKER_ENV=production +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true + +# LANGUAGE_CODE=it-it +# LANGUAGES=(('en-us','English'),('it-it','Italiano')) + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=geonode + +# ################# +# backend +# ################# +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +GEONODE_DATABASE=geonode +GEONODE_DATABASE_USER=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_USER=geonode +GEONODE_GEODATABASE_PASSWORD=geonode +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=localhost +DATABASE_PORT=5432 +DATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode +GEODATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=amqp://admin:admin@localhost:5672// +CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler +ASYNC_SIGNALS=False + +SITEURL=http://localhost:8000/ + +ALLOWED_HOSTS="['django', '*']" + +# Data Uploader +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False + +# ################# +# nginx +# HTTPD Server +# ################# +GEONODE_LB_HOST_IP=django +GEONODE_LB_PORT=8000 +GEOSERVER_LB_HOST_IP=geoserver +GEOSERVER_LB_PORT=8080 +NGINX_BASE_URL=https://localhost + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST=localhost +HTTPS_HOST= + +HTTP_PORT=80 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE=disabled +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# ################# +# geoserver +# ################# +GEOSERVER_WEB_UI_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_PUBLIC_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD=geoserver + +OGC_REQUEST_TIMEOUT=30 +OGC_REQUEST_MAX_RETRIES=1 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost:8080/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' + +# ################# +# Security +# ################# +# Admin Settings +# +# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time +# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, +# otherwise the invoke updateadmin task will be skipped and the current password already stored +# in DB will honored. + +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +ADMIN_EMAIL=admin@localhost + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='GeoNode ' + +# Session/Access Control +LOCKDOWN_GEONODE=False +X_FRAME_OPTIONS="SAMEORIGIN" +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_VIEW_PERMISSION=True +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True + +CORS_ALLOW_ALL_ORIGINS=True +GEOSERVER_CORS_ENABLED=True +GEOSERVER_CORS_ALLOWED_ORIGINS=* +GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS +GEOSERVER_CORS_ALLOWED_HEADERS=* + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_EMAIL_REQUIRED=True +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False +ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False +ACCOUNT_AUTHENTICATION_METHOD=username_email +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV +OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 + +# GeoNode APIs +API_LOCKDOWN=False +TASTYPIE_APIKEY= + +# ################# +# Production and +# Monitoring +# ################# +DEBUG=False + +SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' + +# STATIC_ROOT=/mnt/volumes/statics/static/ +# MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +# GEOIP_PATH=/mnt/volumes/statics/geoip.db + +CACHE_BUSTING_STATIC_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_LOCATION=127.0.0.1:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 +# +# Options for memcached binary, e.g. -vvv to log all requests and cache hits +# +MEMCACHED_OPTIONS= + +MAX_DOCUMENT_SIZE=200 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=False +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# Advanced Workflow +RESOURCE_PUBLISHING=False +ADMIN_MODERATE_UPLOADS=False + +# PostgreSQL +POSTGRESQL_MAX_CONNECTIONS=200 + +# Common containers restart policy +RESTART_POLICY_CONDITION="on-failure" +RESTART_POLICY_DELAY="5s" +RESTART_POLICY_MAX_ATTEMPTS="3" +RESTART_POLICY_WINDOW=120s + +DEFAULT_MAX_UPLOAD_SIZE=5368709120 +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 diff --git a/.env_test b/.env_test new file mode 100644 index 0000000..8026c6c --- /dev/null +++ b/.env_test @@ -0,0 +1,240 @@ +COMPOSE_PROJECT_NAME=geonode +# See https://github.com/containers/podman/issues/13889 +# DOCKER_BUILDKIT=0 +DOCKER_ENV=production +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true + +# LANGUAGE_CODE=it-it +# LANGUAGES=(('en-us','English'),('it-it','Italiano')) + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=geonode + +# ################# +# backend +# ################# +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +GEONODE_DATABASE=geonode +GEONODE_DATABASE_USER=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_USER=geonode_data +GEONODE_GEODATABASE_PASSWORD=geonode_data +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=db +DATABASE_PORT=5432 +DATABASE_URL=postgis://geonode:geonode@db:5432/geonode +GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=amqp://guest:guest@rabbitmq:5672/ +CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler +ASYNC_SIGNALS=True + +SITEURL=http://localhost:8000/ + +ALLOWED_HOSTS="['django', 'localhost', '127.0.0.1']" + +# Data Uploader +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False + +# ################# +# nginx +# HTTPD Server +# ################# +GEONODE_LB_HOST_IP=django +GEONODE_LB_PORT=8000 +GEOSERVER_LB_HOST_IP=geoserver +GEOSERVER_LB_PORT=8080 +NGINX_BASE_URL=http://localhost + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST=localhost +HTTPS_HOST= + +HTTP_PORT=8000 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE=disabled +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# ################# +# geoserver +# ################# +GEOSERVER_WEB_UI_LOCATION=http://localhost:8000/geoserver/ +GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ +GEOSERVER_LOCATION=http://geoserver:8080/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD=geoserver + +OGC_REQUEST_TIMEOUT=5 +OGC_REQUEST_MAX_RETRIES=0 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# ################# +# catalogue +# ################# +CATALOGUE_ENGINE=geonode.catalogue.backends.pycsw_local +CATALOGUE_URL=http://localhost:8000/catalogue/csw + +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' + +# ################# +# Security +# ################# +# Admin Settings +# +# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time +# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, +# otherwise the invoke updateadmin task will be skipped and the current password already stored +# in DB will honored. + +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +ADMIN_EMAIL=admin@localhost + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='GeoNode ' + +# Session/Access Control +LOCKDOWN_GEONODE=False +X_FRAME_OPTIONS="SAMEORIGIN" +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_VIEW_PERMISSION=True +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True + +CORS_ALLOW_ALL_ORIGINS=True +GEOSERVER_CORS_ENABLED=True +GEOSERVER_CORS_ALLOWED_ORIGINS=* +GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS +GEOSERVER_CORS_ALLOWED_HEADERS=* + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_EMAIL_REQUIRED=True +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False +ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False +ACCOUNT_AUTHENTICATION_METHOD=username_email +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV +OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 + +SOCIALACCOUNT_OIDC_PROVIDER_ENABLED=True +SOCIALACCOUNT_PROVIDER=google + +# GeoNode APIs +API_LOCKDOWN=False +TASTYPIE_APIKEY= + +# ################# +# Production and +# Monitoring +# ################# +DEBUG=False + +SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' + +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db + +CACHE_BUSTING_STATIC_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_LOCATION=memcached:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 +# +# Options for memcached binary, e.g. -vvv to log all requests and cache hits +# +MEMCACHED_OPTIONS= + +MAX_DOCUMENT_SIZE=200 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=False +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# Advanced Workflow +RESOURCE_PUBLISHING=False +ADMIN_MODERATE_UPLOADS=False + +# PostgreSQL +POSTGRESQL_MAX_CONNECTIONS=200 + +# Common containers restart policy +RESTART_POLICY_CONDITION="on-failure" +RESTART_POLICY_DELAY="5s" +RESTART_POLICY_MAX_ATTEMPTS="3" +RESTART_POLICY_WINDOW=120s + +DEFAULT_MAX_UPLOAD_SIZE=5368709120 +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=100 + +# Azure AD +MICROSOFT_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_SECRET_KEY= +AZURE_KEY= diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b8b47a6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.py text eol=lf +celery-cmd text eol=lf +./scripts/celery/celeryd text eol=lf diff --git a/.gitignore b/.gitignore index b7faf40..a989960 100644 --- a/.gitignore +++ b/.gitignore @@ -1,207 +1,104 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[codz] -*$py.class +# build artifacts +*.pyc +*.egg-info +build +target +pip-log.txt +MANIFEST +dist +.idea/ +.vscode/ +.pytest_cache/ +pip-wheel-metadata/ +# supporting stuff pulled in by paver script +geonode/development.db +/geoserver/ +downloaded +package-lock.json +GeoIPCities.dat + +docs/i18n/pot/ + +# Static media handling +*.jshintrc +geonode/static/.components/ +geonode/static_root/ +node_modules +# Listing all of them instead of putting geonode/static/lib out of version control. +geonode/local_settings.py + +# Uploaded files +geonode/uploaded +geonode/assets_data + +#Testing output +.coverage +.noseids +.cache +geonode_test_data/ +nosetests.xml +geonode/static/geonode/junit.xml -# C extensions -*.so +#Temporary files +*~ +*.swp +celerybeat-schedule.* +*.log -# Distribution / packaging +# OSX specific .Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST +.DS_Store + +# Docs +docs/_build + +# ec2.py +scripts/cloud/.gnec2.cfg +uploaded +development.db +master +slave + +# virtual env +venv +geonode/bin/ +geonode/djcelery/ +geonode/include/ +geonode/lib/ +geonode/local/ +geonode/man/ +geonode/pip-selfcheck.json +geonode/shapely/ +geonode/share/ + +# celery files +celerybeat-schedule.* + +geonode\.tests\.bdd\.e2e\.test_login/ -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec +*.log -# Installer logs -pip-log.txt -pip-delete-this-directory.txt +/celerybeat.pid +/celerybeat-schedule -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ -.pytest_cache/ -cover/ +scripts/spcgeonode/_volume_* -# Translations -*.mo -*.pot +/geonode/test_settings.py -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock -#poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -#pdm.lock -#pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -#pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments +/geonode/static/yarn.lock + +/geonode/development.db-journal + +/output.bin + +/worker.state + +# Docker Hub hooks +!hooks/* + +# Environment files (contengono credenziali sensibili) .env -.envrc -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -# .vscode/ - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc - -# Cursor -# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to -# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data -# refer to https://docs.cursor.com/context/ignore-files -.cursorignore -.cursorindexingignore - -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ + +# Template deve essere committato per setup automatico +!.env.template + diff --git a/.tx/config b/.tx/config new file mode 100644 index 0000000..ddad00c --- /dev/null +++ b/.tx/config @@ -0,0 +1,38 @@ +[main] +host = https://www.transifex.com + +[geonode.master] +file_filter = geonode/locale//LC_MESSAGES/django.po +source_file = geonode/locale/en/LC_MESSAGES/django.po +source_lang = en +type = PO + +[geonode.javascript] +file_filter = geonode/locale//LC_MESSAGES/djangojs.po +source_file = geonode/locale/en/LC_MESSAGES/djangojs.po +source_lang = en +type = PO + +#[geonode.docs-index] +#file_filter = docs/i18n//LC_MESSAGES/index.po +#source_file = docs/i18n/en/LC_MESSAGES/index.po +#source_lang = en +#type = PO + +#[geonode.docs-organizational] +#file_filter = docs/i18n//LC_MESSAGES/organizational.po +#source_file = docs/i18n/en/LC_MESSAGES/organizational.po +#source_lang = en +#type = PO + +#[geonode.docs-reference] +#file_filter = docs/i18n//LC_MESSAGES/reference.po +#source_file = docs/i18n/en/LC_MESSAGES/reference.po +#source_lang = en +#type = PO + +#[geonode.docs-tutorials] +#file_filter = docs/i18n//LC_MESSAGES/tutorials.po +#source_file = docs/i18n/en/LC_MESSAGES/tutorials.po +#source_lang = en +#type = PO diff --git a/CLA_INDIVIDUAL.md b/CLA_INDIVIDUAL.md new file mode 100644 index 0000000..ad23ab2 --- /dev/null +++ b/CLA_INDIVIDUAL.md @@ -0,0 +1,36 @@ + +# Agreement for GeoNode/geonode + +OSGeo Individual Contributor License Agreement (“Agreement”) v1.0 + +Thank you for your interest in The Open Source Geospatial Foundation (the “Foundation”). In order to clarify the intellectual property license granted with Contributions from any person or entity, the Foundation must have a Contributor License Agreement (“CLA”) on file that has been signed by each Contributor, indicating agreement to the license terms below. This license is for your protection as a Contributor as well as the protection of the Foundation and its users; it does not change your rights to use your own Contributions for any other purpose. If you have not already done so, please complete and send an original signed Agreement to The Open Source Geospatial Foundation, [address]. If necessary, you may send it by facsimile to the Foundation at [fax]. Please read this document carefully before signing and keep a copy for your records. + +You accept and agree to the following terms and conditions for Your present and future Contributions submitted to the Foundation. In return, the Foundation shall not use Your Contributions in a way that is contrary to the public benefit or inconsistent with its nonprofit status and bylaws in effect at the time of the Contribution. Except for the license granted herein to the Foundation and recipients of software distributed by the Foundation, You reserve all right, title, and interest in and to Your Contributions. + +1. Definitions. “You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with the Foundation. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. “Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to the Foundation for inclusion in, or documentation of, any of the products owned or managed by the Foundation (the “Work”). For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to the Foundation or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Foundation for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as “Not a Contribution.” + +2. Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to the Foundation and to recipients of software distributed by the Foundation a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works. + +3. Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to the Foundation and to recipients of software distributed by the Foundation a perpetual, worldwide, non-exclusive, fully-paid, royalty-free, irrevocable (except as stated in this section) patent license to have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) were submitted, as such Work existed as of the date of your Contribution(s). This patent license shall not apply to any other combinations which include the Contribution(s). If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed. + +4. You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to the Foundation, or that your employer has executed a separate Corporate CLA with the Foundation. + +5. You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions. + +6. You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + +7. Should You wish to submit work that is not Your original creation, You may submit it to the Foundation separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as “Submitted on behalf of a third-party:”. + +8. You agree to notify the Foundation of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect. + +`Name: ___________________________________________` + +`E-mail: _________________________________________` + +`Date: ___________________________________________` + +`Corporate Contributor Information: ______________` + +`_________________________________________________` + +If you are employed as a software engineer, or if your employer is in the business of developing software, or otherwise may claim rights in the Contributions, please provide information about your employer's policy on contributing to open source projects, including the name of the supervisor to contact in connection with such contributions. \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..6402b4d --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team via alessio.fabiani@geosolutionsgroup.com . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1e52f55 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM geonode/geonode-base:latest-ubuntu-22.04 + +# copy local geonode src inside container +COPY . /usr/src/geonode/ +WORKDIR /usr/src/geonode + +COPY wait-for-databases.sh /usr/bin/wait-for-databases +RUN chmod +x /usr/bin/wait-for-databases +RUN chmod +x /usr/src/geonode/tasks.py \ + && chmod +x /usr/src/geonode/entrypoint.sh + +COPY celery.sh /usr/bin/celery-commands +RUN chmod +x /usr/bin/celery-commands + +COPY celery-cmd /usr/bin/celery-cmd +RUN chmod +x /usr/bin/celery-cmd + +RUN yes w | pip install --src /usr/src -r requirements.txt &&\ + yes w | pip install -e . + +# Cleanup apt update lists +RUN apt-get autoremove --purge &&\ + apt-get clean &&\ + rm -rf /var/lib/apt/lists/* + +# Export ports +EXPOSE 8000 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..fbbc2e7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include README README.rst README.md LICENSE requirements.txt +recursive-exclude geonode/static/.components +recursive-include geonode *.css *.eot *.gif *.html *.ico *.ini *.jpg *.js *.json *.less *.md *.mo *.po *.otf *.png *.rst *.sample *.svg *.swf *.ttf *.txt *.woff *.woff2 *.xml *.xsl README +recursive-include geonode/contrib/geosites gunicorn nginx diff --git a/README.md b/README.md new file mode 100644 index 0000000..187d0da --- /dev/null +++ b/README.md @@ -0,0 +1,185 @@ +![GeoNode](https://raw.githubusercontent.com/GeoNode/documentation/master/about/img/geonode-logo_for_readme.gif "GeoNode") +![OSGeo Project](https://www.osgeo.cn/qgis/_static/images/osgeoproject.png) + +Table of Contents +================= + +- [Table of Contents](#table-of-contents) + - [What is GeoNode?](#what-is-geonode) + - [Try out GeoNode](#try-out-geonode) + - [Quick Docker Start](#quick-docker-start) + - [Learn GeoNode](#learn-geonode) + - [Development](#development) + - [Contributing](#contributing) + - [Roadmap](#roadmap) + - [Showcase](#showcase) + - [Most useful links](#most-useful-links) + - [Licensing](#licensing) + +What is GeoNode? +---------------- + +GeoNode is a geospatial content management system, a platform for the +management and publication of geospatial data. It brings together mature +and stable open-source software projects under a consistent and +easy-to-use interface allowing non-specialized users to share data and +create interactive maps. + +Data management tools built into GeoNode allow for integrated creation +of data, metadata, and map visualization. Each dataset in the system can +be shared publicly or restricted to allow access to only specific users. +Social features like user profiles and commenting and rating systems +allow for the development of communities around each platform to +facilitate the use, management, and quality control of the data the +GeoNode instance contains. + +It is also designed to be a flexible platform that software developers +can extend, modify or integrate against to meet requirements in their +own applications. + +Try out GeoNode +--------------- + +If you just want to try out GeoNode visit our official Demo online at: +[https://development.demo.geonode.org](https://development.demo.geonode.org). After your registration, you will be able +to test all basic functionalities like uploading layers, creation of +maps, editing metadata, styles, and much more. To get an overview what +GeoNode can do we recommend having a look at the [Users +Workshop](https://docs.geonode.org/en/master/usage/index.html). + +Quick Docker Start +------------------ + + ```bash + python create-envfile.py + ``` +`create-envfile.py` accepts the following arguments: + +- `--https`: Enable SSL. It's disabled by default +- `--env_type`: + - When set to `prod` `DEBUG` is disabled and the creation of a valid `SSL` is requested to Letsencrypt's ACME server + - When set to `test` `DEBUG` is disabled and a test `SSL` certificate is generated for local testing + - When set to `dev` `DEBUG` is enabled and no `SSL` certificate is generated +- `--hostname`: The URL that will serve GeoNode (`localhost` by default) +- `--email`: The administrator's email. Notice that a real email and valid SMPT configurations are required if `--env_type` is set to `prod`. Letsencrypt uses email for issuing the SSL certificate +- `--geonodepwd`: GeoNode's administrator password. A random value is set if left empty +- `--geoserverpwd`: GeoNode's administrator password. A random value is set if left empty +- `--pgpwd`: PostgreSQL's administrator password. A random value is set if left empty +- `--dbpwd`: GeoNode DB user role's password. A random value is set if left empty +- `--geodbpwd`: GeoNode data DB user role's password. A random value is set if left empty +- `--clientid`: Client id of Geoserver's GeoNode Oauth2 client. A random value is set if left empty +- `--clientsecret`: Client secret of Geoserver's GeoNode Oauth2 client. A random value is set if left empty + +```bash + docker compose build + docker compose up -d +``` + +Learn GeoNode +------------- + +After you´ve finished the setup process make yourself familiar with the +general usage and settings of your GeoNodes instance. - the [User +Training](https://docs.geonode.org/en/master/usage/index.html) +is going in depth into what we can do. - the [Administrators +Workshop](https://docs.geonode.org/en/master/admin/index.html) +will guide you to the most important parts regarding management commands +and configuration settings. + +Development +----------- + +GeoNode is a web-based GIS tool, and as such, in order to do development +on GeoNode itself or to integrate it into your own application, you +should be familiar with basic web development concepts as well as with +general GIS concepts. + +For development, GeoNode can be run in a 'development environment'. In +contrast to a 'production environment' development differs as it uses +lightweight components to speed up things. + +To get started visit the [Developer +workshop](https://docs.geonode.org/en/master/devel/index.html) +for a basic overview. + +If you're planning to customize your GeoNode instance or to extend +its functionalities it's not advisable to change core files in any +case. In this case, it's common to setup a [GeoNode Project +Template](https://github.com/GeoNode/geonode-project). + +Contributing +------------ + +GeoNode is an open source project and contributors are needed to keep +this project moving forward. Learn more on how to contribute on our +[Community +Bylaws](https://github.com/GeoNode/geonode/wiki/Community-Bylaws). + +Roadmap +------- + +GeoNode's development roadmap is documented in a series of GeoNode +Improvement Projects (GNIPS). They are documented at [GeoNode Wiki](https://github.com/GeoNode/geonode/wiki/GeoNode-Improvement-Proposals). + +GNIPS are considered to be large undertakings that will add a large +number of features to the project. As such they are the topic of +community discussion and guidance. The community discusses these on the +developer mailing list: http://lists.osgeo.org/pipermail/geonode-devel/ + +Showcase +-------- + +A handful of other Open Source projects extend GeoNode’s functionality +by tapping into the re-usability of Django applications. Visit our +gallery to see how the community uses GeoNode: [GeoNode +Showcase](https://geonode.org/gallery/). + +The development community is very supportive of new projects and +contributes ideas and guidance for newcomers. + +Most useful links +----------------- + + +**General** + +- Project homepage: https://geonode.org +- Repository: https://github.com/GeoNode/geonode +- Official Demos: https://demo.geonode.org +- GeoNode Wiki: https://github.com/GeoNode/geonode/wiki +- Issue tracker: https://github.com/GeoNode/geonode-project/issues + + In case of sensitive bugs like security vulnerabilities, please + contact a GeoNode Core Developer directly instead of using an issue + tracker. We value your effort to improve the security and privacy of + this project! + +**Related projects** + +- GeoNode Project: https://github.com/GeoNode/geonode-project +- GeoNode at Docker: https://hub.docker.com/u/geonode +- GeoNode OSGeo-Live: https://live.osgeo.org/en/ + + +**Support** + +- User Mailing List: https://lists.osgeo.org/cgi-bin/mailman/listinfo/geonode-users +- Developer Mailing List: https://lists.osgeo.org/cgi-bin/mailman/listinfo/geonode-devel +- Gitter Chat: https://gitter.im/GeoNode/general + + +Licensing +--------- + +GeoNode is Copyright 2018 Open Source Geospatial Foundation (OSGeo). + +GeoNode is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your +option) any later version. GeoNode is distributed in the hope that it +will be useful, but WITHOUT ANY WARRANTY; without even the implied +warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with GeoNode. If not, see http://www.gnu.org/licenses. diff --git a/celery-cmd b/celery-cmd new file mode 100644 index 0000000..a600163 --- /dev/null +++ b/celery-cmd @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# A configurable celery command. +# Luca Pasquali +CELERY_BIN=${CELERY_BIN:-"$(which celery||echo celery)"} +CELERY_APP=${CELERY_APP:-"geonode.celery_app:app"} +CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker@%h.state"} +# expressed in KB +CELERY__MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"200000"} +CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"15,10"} +CELERY__MAX_TASKS_PER_CHILD=${CELERY__MAX_TASKS_PER_CHILD:-"10"} +CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -B -E"} +CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"celery.beat:PersistentScheduler"} +CELERY__LOG_LEVEL=${CELERY__LOG_LEVEL:-"INFO"} +CELERY__LOG_FILE=${CELERY__LOG_FILE:-"/var/log/celery.log"} +CELERY__WORKER_NAME=${CELERY__WORKER_NAME:-"worker1@%h"} +CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"} + +$CELERY_BIN -A $CELERY_APP worker --autoscale=$CELERY__AUTOSCALE_VALUES \ + --max-memory-per-child=$CELERY__MAX_MEMORY_PER_CHILD $CELERY__OPTS \ + --statedb=$CELERY__STATE_DB --scheduler=$CELERY__BEAT_SCHEDULE \ + --loglevel=$CELERY__LOG_LEVEL -n $CELERY__WORKER_NAME -f $CELERY__LOG_FILE \ + --concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD diff --git a/celery.sh b/celery.sh new file mode 100644 index 0000000..db81672 --- /dev/null +++ b/celery.sh @@ -0,0 +1,3 @@ +#!/bin/bash +nohup celery -A geonode.celery_app:app beat -l DEBUG -f /var/log/celery.log &>/dev/null & +nohup celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state --scheduler=celery.beat:PersistentScheduler --loglevel=INFO --concurrency=2 -n worker1@%h -f /var/log/celery.log &>/dev/null \ No newline at end of file diff --git a/create-envfile.py b/create-envfile.py new file mode 100644 index 0000000..503b58f --- /dev/null +++ b/create-envfile.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2022 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import argparse +import json +import logging +import os +import random +import re +import string +import sys +import ast + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +logger = logging.getLogger() +handler = logging.StreamHandler(sys.stdout) +logger.setLevel(logging.INFO) +formatter = logging.Formatter("%(levelname)s - %(message)s") +handler.setFormatter(formatter) +logger.addHandler(handler) + + +def shuffle(chars): + chars_as_list = list(chars) + random.shuffle(chars_as_list) + return "".join(chars_as_list) + +_simple_chars = shuffle(string.ascii_letters + string.digits) +_strong_chars = shuffle(string.ascii_letters + string.digits + "#%*._~") + +def generate_env_file(args): + # validity checks + if not os.path.exists(args.sample_file): + logger.error(f"File does not exists {args.sample_file}") + raise FileNotFoundError + + if args.file and not os.path.isfile(args.file): + logger.error(f"File does not exists: {args.file}") + raise FileNotFoundError + + if args.https and not args.email: + raise Exception("With HTTPS enabled, the email parameter is required") + + _sample_file = None + with open(args.sample_file, "r+") as sample_file: + _sample_file = sample_file.read() + + if not _sample_file: + raise Exception("Sample file is empty!") + + def _get_vals_to_replace(args): + _config = ["sample_file", "file", "env_type", "https", "email"] + _jsfile = {} + if args.file: + with open(args.file) as _json_file: + _jsfile = json.load(_json_file) + + _vals_to_replace = {key: _jsfile.get(key, val) for key, val in vars(args).items() if key not in _config} + tcp = "https" if ast.literal_eval(f"{_jsfile.get('https', args.https)}".capitalize()) else "http" + + _vals_to_replace["public_port"] = ( + "443" if ast.literal_eval(f"{_jsfile.get('https', args.https)}".capitalize()) else "80" + ) + _vals_to_replace["http_host"] = _jsfile.get("hostname", args.hostname) if tcp == "http" else "" + _vals_to_replace["https_host"] = _jsfile.get("hostname", args.hostname) if tcp == "https" else "" + + _vals_to_replace["siteurl"] = f"{tcp}://{_jsfile.get('hostname', args.hostname)}" + _vals_to_replace["secret_key"] = _jsfile.get("secret_key", args.secret_key) or "".join( + random.choice(_strong_chars) for _ in range(50) + ) + _vals_to_replace["letsencrypt_mode"] = ( + "disabled" + if not _vals_to_replace.get("https_host") + else "staging" + if _jsfile.get("env_type", args.env_type) in ["test"] + else "production" + ) + _vals_to_replace["debug"] = False if _jsfile.get("env_type", args.env_type) in ["prod", "test"] else True + _vals_to_replace["email"] = _jsfile.get("email", args.email) + + if tcp == "https" and not _vals_to_replace["email"]: + raise Exception("With HTTPS enabled, the email parameter is required") + + return {**_jsfile, **_vals_to_replace} + + for key, val in _get_vals_to_replace(args).items(): + _val = val or "".join(random.choice(_simple_chars) for _ in range(15)) + if isinstance(val, bool) or key in ["email", "http_host", "https_host"]: + _val = str(val) + _sample_file = re.sub( + "{" + key + "}", + lambda _: _val, + _sample_file, + ) + + with open(f"{dir_path}/.env", "w+") as output_env: + output_env.write(_sample_file) + logger.info(f".env file created: {dir_path}/.env") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="ENV file builder", + description="Tool for generate environment file automatically. The information can be passed or via CLI or via JSON file ( --file /path/env.json)", + usage="python create-envfile.py localhost -f /path/to/json/file.json", + allow_abbrev=False, + ) + parser.add_argument( + "--noinput", + "--no-input", + action="store_false", + dest="confirmation", + help=("skips prompting for confirmation."), + ) + parser.add_argument( + "-hn", + "--hostname", + help=f"Host name, default localhost", + default="localhost", + ) + + # expected path as a value + parser.add_argument( + "-sf", + "--sample_file", + help=f"Path of the sample file to use as a template. Default is: {dir_path}/.env.sample", + default=f"{dir_path}/.env.sample", + ) + parser.add_argument( + "-f", + "--file", + help="absolute path of the file with the configuration. Note: we expect that the keys of the dictionary have the same name as the CLI params", + ) + # booleans + parser.add_argument("--https", action="store_true", default=False, help="If provided, https is used") + # strings + parser.add_argument("--email", help="Admin email, this field is required if https is enabled") + + parser.add_argument("--geonodepwd", help="GeoNode admin password") + parser.add_argument("--geoserverpwd", help="Geoserver admin password") + parser.add_argument("--pgpwd", help="PostgreSQL password") + parser.add_argument("--dbpwd", help="GeoNode DB user password") + parser.add_argument("--geodbpwd", help="Geodatabase user password") + parser.add_argument("--clientid", help="Oauth2 client id") + parser.add_argument("--clientsecret", help="Oauth2 client secret") + parser.add_argument("--secret_key", help="Django Secret Key") + + parser.add_argument( + "--env_type", + help="Development/production or test", + choices=["prod", "test", "dev"], + default="prod", + ) + + args = parser.parse_args() + + if not args.confirmation: + generate_env_file(args) + else: + overwrite_env = input("This action will overwrite any existing .env file. Do you wish to continue? (y/n)") + if overwrite_env not in ["y", "n"]: + logger.error("Please enter a valid response") + if overwrite_env == "y": + generate_env_file(args) diff --git a/dev_config.yml b/dev_config.yml new file mode 100644 index 0000000..3fa02cd --- /dev/null +++ b/dev_config.yml @@ -0,0 +1,8 @@ +--- +GEOSERVER_URL: "https://artifacts.geonode.org/geoserver/2.23.0/geoserver.war" +DATA_DIR_URL: "https://artifacts.geonode.org/geoserver/2.23.0/geonode-geoserver-ext-web-app-data.zip" +JETTY_RUNNER_URL: "https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.31.v20200723/jetty-runner-9.4.31.v20200723.jar" +WINDOWS: + py2exe: "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe" + pyproj: "https://pyproj.googlecode.com/files/pyproj-1.9.3.win32-py2.7.exe" + lxml: "https://pypi.python.org/packages/2.7/l/lxml/lxml-3.6.0.win32-py2.7.exe" diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..6485d85 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Exit script in case of error +set -e + +INVOKE_LOG_STDOUT=${INVOKE_LOG_STDOUT:-FALSE} +invoke () { + if [ $INVOKE_LOG_STDOUT = 'true' ] || [ $INVOKE_LOG_STDOUT = 'True' ] + then + /usr/local/bin/invoke $@ + else + /usr/local/bin/invoke $@ > /usr/src/geonode/invoke.log 2>&1 + fi + echo "$@ tasks done" +} + +# Start cron && memcached services +service cron restart + +echo $"\n\n\n" +echo "-----------------------------------------------------" +echo "STARTING DJANGO ENTRYPOINT $(date)" +echo "-----------------------------------------------------" + +invoke update + +source $HOME/.bashrc +source $HOME/.override_env + +echo DOCKER_API_VERSION=$DOCKER_API_VERSION +echo POSTGRES_USER=$POSTGRES_USER +echo POSTGRES_PASSWORD=$POSTGRES_PASSWORD +echo DATABASE_URL=$DATABASE_URL +echo GEODATABASE_URL=$GEODATABASE_URL +echo SITEURL=$SITEURL +echo ALLOWED_HOSTS=$ALLOWED_HOSTS +echo GEOSERVER_PUBLIC_LOCATION=$GEOSERVER_PUBLIC_LOCATION +echo MONITORING_ENABLED=$MONITORING_ENABLED +echo MONITORING_HOST_NAME=$MONITORING_HOST_NAME +echo MONITORING_SERVICE_NAME=$MONITORING_SERVICE_NAME +echo MONITORING_DATA_TTL=$MONITORING_DATA_TTL + +# invoke waitfordbs + +cmd="$@" + +if [ ${IS_CELERY} = "true" ] || [ ${IS_CELERY} = "True" ] +then + echo "Executing Celery server $cmd for Production" +else + + invoke migrations + invoke prepare + + if [ ${FORCE_REINIT} = "true" ] || [ ${FORCE_REINIT} = "True" ] || [ ! -e "/mnt/volumes/statics/geonode_init.lock" ]; then + invoke fixtures + invoke monitoringfixture + invoke initialized + invoke updateadmin + fi + + # Setup Keycloak Social App if OIDC is enabled (sempre eseguito) + if [ "${SOCIALACCOUNT_OIDC_PROVIDER_ENABLED}" = "true" ] || [ "${SOCIALACCOUNT_OIDC_PROVIDER_ENABLED}" = "True" ]; then + echo "🔐 Setting up Keycloak Social App..." + python manage.py setup_keycloak --no-input || echo "⚠️ Keycloak setup failed or already configured" + fi + + invoke statics + + echo "Executing UWSGI server $cmd for Production" +fi + +echo "-----------------------------------------------------" +echo "FINISHED DJANGO ENTRYPOINT --------------------------" +echo "-----------------------------------------------------" + +# Run the CMD +echo "got command $cmd" +exec $cmd diff --git a/flake8.sh b/flake8.sh new file mode 100644 index 0000000..5b7fab0 --- /dev/null +++ b/flake8.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +flake8 geonode diff --git a/flake8.txt b/flake8.txt new file mode 100644 index 0000000..3063e84 --- /dev/null +++ b/flake8.txt @@ -0,0 +1,7 @@ +# Check formatting with + +flake8 --config=setup.cfg + +# Auto-fix formatting with + +autopep8 --global-config setup.cfg -i -a -a [files [files ...]] diff --git a/geonode/__init__.py b/geonode/__init__.py new file mode 100644 index 0000000..4c56c53 --- /dev/null +++ b/geonode/__init__.py @@ -0,0 +1,42 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os + +__version__ = (4, 4, 0, "dev", 0) + + +def get_version(): + import geonode.version + + return geonode.version.get_version(__version__) + + +def main(global_settings, **settings): + from django.core.wsgi import get_wsgi_application + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings.get("django_settings")) + app = get_wsgi_application() + return app + + +class GeoNodeException(Exception): + """Base class for exceptions in this module.""" + + pass diff --git a/geonode/api/__init__.py b/geonode/api/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/api/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/api/api.py b/geonode/api/api.py new file mode 100644 index 0000000..316510b --- /dev/null +++ b/geonode/api/api.py @@ -0,0 +1,758 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import time + +from django.apps import apps +from django.db.models import Q +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group +from django.urls import reverse +from django.contrib.contenttypes.models import ContentType +from django.conf import settings +from django.db.models import Count +from django.utils.translation import get_language + +from avatar.templatetags.avatar_tags import avatar_url + +from geonode import geoserver +from geonode.api.paginator import CrossSiteXHRPaginator +from geonode.api.authorization import ( + GeoNodeStyleAuthorization, + ApiLockdownAuthorization, + GroupAuthorization, + GroupProfileAuthorization, + GeoNodePeopleAuthorization, +) +from guardian.shortcuts import get_objects_for_user +from tastypie.bundle import Bundle + +from geonode.base.models import ResourceBase, ThesaurusKeyword +from geonode.base.models import TopicCategory +from geonode.base.models import Region +from geonode.base.models import HierarchicalKeyword +from geonode.base.models import ThesaurusKeywordLabel +from geonode.layers.models import Dataset, Style +from geonode.people.utils import get_available_users +from geonode.maps.models import Map +from geonode.geoapps.models import GeoApp +from geonode.documents.models import Document +from geonode.groups.models import GroupProfile, GroupCategory +from django.core.serializers.json import DjangoJSONEncoder +from tastypie.serializers import Serializer +from tastypie import fields +from tastypie.resources import ModelResource +from tastypie.constants import ALL, ALL_WITH_RELATIONS + +from geonode.utils import check_ogc_backend +from geonode.security.utils import get_visible_resources + +FILTER_TYPES = {"dataset": Dataset, "map": Map, "document": Document, "geoapp": GeoApp} + + +class CountJSONSerializer(Serializer): + """Custom serializer to post process the api and add counts""" + + def get_resources_counts(self, options): + if settings.SKIP_PERMS_FILTER: + resources = ResourceBase.objects.all() + else: + resources = get_objects_for_user(options["user"], "base.view_resourcebase") + + resources = get_visible_resources( + resources, + options["user"], + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + subtypes = [] + if resources and resources.exists(): + if options["title_filter"]: + resources = resources.filter(title__icontains=options["title_filter"]) + if options["type_filter"]: + _type_filter = options["type_filter"] + + for label, app in apps.app_configs.items(): + if hasattr(app, "type") and app.type == "GEONODE_APP": + if hasattr(app, "default_model"): + _model = apps.get_model(label, app.default_model) + if issubclass(_model, _type_filter): + subtypes.append(resources.filter(polymorphic_ctype__model=_model.__name__.lower())) + + if not isinstance(_type_filter, str): + _type_filter = _type_filter.__name__.lower() + resources = resources.filter(polymorphic_ctype__model=_type_filter) + + counts = list() + if subtypes: + for subtype in subtypes: + counts.extend(list(subtype.values(options["count_type"]).annotate(count=Count(options["count_type"])))) + else: + counts = list(resources.values(options["count_type"]).annotate(count=Count(options["count_type"]))) + + _counts = {} + for c in counts: + if c and c["count"] and options["count_type"]: + if not _counts.get(c[options["count_type"]], None): + _counts.update({c[options["count_type"]]: c["count"]}) + else: + _counts[c[options["count_type"]]] += c["count"] + return _counts + + def to_json(self, data, options=None): + options = options or {} + data = self.to_simple(data, options) + counts = self.get_resources_counts(options) + if "objects" in data: + for item in data["objects"]: + item["count"] = counts.get(item["id"], 0) + # Add in the current time. + data["requested_time"] = time.time() + + return json.dumps(data, cls=DjangoJSONEncoder, sort_keys=True) + + +class TypeFilteredResource(ModelResource): + """Common resource used to apply faceting to categories, keywords, and + regions based on the type passed as query parameter in the form + type:dataset/map/document""" + + count = fields.IntegerField() + + def build_filters(self, filters=None, ignore_bad_filters=False): + if filters is None: + filters = {} + self.type_filter = None + self.title_filter = None + + orm_filters = super().build_filters(filters) + + if "type" in filters and filters["type"] in FILTER_TYPES.keys(): + self.type_filter = FILTER_TYPES[filters["type"]] + else: + self.type_filter = None + if "title__icontains" in filters: + self.title_filter = filters["title__icontains"] + + return orm_filters + + def serialize(self, request, data, format, options=None): + if options is None: + options = {} + options["title_filter"] = getattr(self, "title_filter", None) + options["type_filter"] = getattr(self, "type_filter", None) + options["user"] = request.user + + return super().serialize(request, data, format, options) + + +class TagResource(TypeFilteredResource): + """Tags api""" + + def serialize(self, request, data, format, options=None): + if options is None: + options = {} + options["count_type"] = "keywords" + + return super().serialize(request, data, format, options) + + class Meta: + queryset = HierarchicalKeyword.objects.all().order_by("name") + resource_name = "keywords" + allowed_methods = ["get"] + filtering = { + "slug": ALL, + } + serializer = CountJSONSerializer() + authorization = ApiLockdownAuthorization() + + +class ThesaurusKeywordResource(TypeFilteredResource): + """ThesaurusKeyword api""" + + thesaurus_identifier = fields.CharField(null=False) + label_id = fields.CharField(null=False) + + def build_filters(self, filters={}, ignore_bad_filters=False): + """adds filtering by current language""" + _filters = filters.copy() + id = _filters.pop("id", None) + orm_filters = super().build_filters(_filters) + + if id is not None: + orm_filters["id__in"] = id + + if "thesaurus" in _filters: + orm_filters["thesaurus__identifier"] = _filters["thesaurus"] + + return orm_filters + + def serialize(self, request, data, format, options={}): + options["count_type"] = "tkeywords__id" + + return super().serialize(request, data, format, options) + + def dehydrate_id(self, bundle): + return bundle.obj.id + + def dehydrate_label_id(self, bundle): + return bundle.obj.id + + def dehydrate_thesaurus_identifier(self, bundle): + return bundle.obj.thesaurus.identifier + + def dehydrate(self, bundle): + lang = get_language() + label = ThesaurusKeywordLabel.objects.filter(keyword=bundle.data["id"]).filter(lang=lang) + if label.exists(): + bundle.data["label_id"] = label.get().id + bundle.data["label"] = label.get().label + bundle.data["alt_label"] = label.get().label + else: + bundle.data["label"] = bundle.data["alt_label"] + + return bundle + + class Meta: + queryset = ThesaurusKeyword.objects.all().order_by("alt_label").select_related("thesaurus") + + resource_name = "thesaurus/keywords" + allowed_methods = ["get"] + filtering = { + "id": ALL, + "alt_label": ALL, + "thesaurus": ALL, + } + serializer = CountJSONSerializer() + authorization = ApiLockdownAuthorization() + + +class RegionResource(TypeFilteredResource): + """Regions api""" + + def serialize(self, request, data, format, options=None): + if options is None: + options = {} + options["count_type"] = "regions" + + return super().serialize(request, data, format, options) + + class Meta: + queryset = Region.objects.all().order_by("name") + resource_name = "regions" + allowed_methods = ["get"] + filtering = { + "name": ALL, + "code": ALL, + } + if settings.API_INCLUDE_REGIONS_COUNT: + serializer = CountJSONSerializer() + authorization = ApiLockdownAuthorization() + + +class TopicCategoryResource(TypeFilteredResource): + """Category api""" + + layers_count = fields.IntegerField(default=0) + + def dehydrate_datasets_count(self, bundle): + request = bundle.request + obj_with_perms = get_objects_for_user(request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="dataset" + ) + filter_set = bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")).filter(metadata_only=False) + + if not settings.SKIP_PERMS_FILTER: + filter_set = get_visible_resources( + filter_set, + request.user if request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + return filter_set.distinct().count() + + def serialize(self, request, data, format, options=None): + if options is None: + options = {} + options["count_type"] = "category" + + return super().serialize(request, data, format, options) + + class Meta: + queryset = TopicCategory.objects.all() + resource_name = "categories" + allowed_methods = ["get"] + filtering = { + "identifier": ALL, + } + serializer = CountJSONSerializer() + authorization = ApiLockdownAuthorization() + + +class GroupCategoryResource(TypeFilteredResource): + detail_url = fields.CharField() + member_count = fields.IntegerField() + resource_counts = fields.CharField() + + class Meta: + queryset = GroupCategory.objects.all() + allowed_methods = ["get"] + include_resource_uri = False + filtering = {"slug": ALL, "name": ALL} + ordering = ["name"] + authorization = ApiLockdownAuthorization() + + def apply_filters(self, request, applicable_filters): + filtered = super().apply_filters(request, applicable_filters) + return filtered + + def dehydrate_detail_url(self, bundle): + return bundle.obj.get_absolute_url() + + def dehydrate_member_count(self, bundle): + request = bundle.request + user = request.user + filtered = bundle.obj.groups.all() + if not user.is_authenticated or user.is_anonymous: + filtered = filtered.exclude(access="private") + elif not user.is_superuser: + categories_ids = user.group_list_all().values("categories") + filtered = filtered.filter(Q(id__in=categories_ids) | ~Q(access="private")) + return filtered.count() + + def dehydrate(self, bundle): + """Provide additional resource counts""" + request = bundle.request + counts = _get_resource_counts( + request, resourcebase_filter_kwargs={"group__groupprofile__categories": bundle.obj} + ) + bundle.data.update(resource_counts=counts) + return bundle + + +class GroupProfileResource(ModelResource): + categories = fields.ToManyField(GroupCategoryResource, "categories", full=True) + member_count = fields.CharField() + manager_count = fields.CharField() + logo_url = fields.CharField() + detail_url = fields.CharField() + + class Meta: + queryset = GroupProfile.objects.all() + resource_name = "group_profile" + allowed_methods = ["get"] + filtering = { + "title": ALL, + "slug": ALL, + "categories": ALL_WITH_RELATIONS, + } + ordering = ["title", "last_modified"] + authorization = GroupProfileAuthorization() + + def dehydrate_member_count(self, bundle): + """Provide relative URL to the geonode UI's page on the group""" + return bundle.obj.member_queryset().count() + + def dehydrate_manager_count(self, bundle): + """Provide relative URL to the geonode UI's page on the group""" + return bundle.obj.get_managers().count() + + def dehydrate_detail_url(self, bundle): + """Return relative URL to the geonode UI's page on the group""" + if bundle.obj.slug: + return reverse("group_detail", args=[bundle.obj.slug]) + else: + return None + + def dehydrate_logo_url(self, bundle): + return bundle.obj.logo_url + + +class GroupResource(ModelResource): + group_profile = fields.ToOneField(GroupProfileResource, "groupprofile", full=True, null=True, blank=True) + resource_counts = fields.CharField() + + class Meta: + queryset = Group.objects.exclude(groupprofile=None) + resource_name = "groups" + allowed_methods = ["get"] + filtering = { + "name": ALL, + "title": ALL, + "group_profile": ALL_WITH_RELATIONS, + } + ordering = ["name", "last_modified"] + authorization = GroupAuthorization() + + def dehydrate(self, bundle): + """Provide additional resource counts""" + request = bundle.request + counts = _get_resource_counts(request, resourcebase_filter_kwargs={"group": bundle.obj, "metadata_only": False}) + + bundle.data.update(resource_counts=counts) + return bundle + + def get_object_list(self, request): + """ + Overridden in order to exclude the ``anoymous`` group from the list + + """ + + qs = super().get_object_list(request) + return qs.exclude(name="anonymous") + + +class ProfileResource(TypeFilteredResource): + """Profile api""" + + avatar_100 = fields.CharField(null=True) + profile_detail_url = fields.CharField() + email = fields.CharField(default="") + layers_count = fields.IntegerField(default=0) + maps_count = fields.IntegerField(default=0) + documents_count = fields.IntegerField(default=0) + current_user = fields.BooleanField(default=False) + activity_stream_url = fields.CharField(null=True) + + def build_filters(self, filters=None, ignore_bad_filters=False): + """adds filtering by group functionality""" + if filters is None: + filters = {} + + orm_filters = super().build_filters(filters) + + if "group" in filters: + orm_filters["group"] = filters["group"] + + if "name__icontains" in filters: + orm_filters["username__icontains"] = filters["name__icontains"] + + return orm_filters + + def apply_filters(self, request, applicable_filters): + """filter by group if applicable by group functionality""" + + group = applicable_filters.pop("group", None) + name = applicable_filters.pop("name__icontains", None) + + semi_filtered = super().apply_filters(request, applicable_filters) + + if group is not None: + semi_filtered = semi_filtered.filter(groupmember__group__slug=group) + + if name is not None: + semi_filtered = semi_filtered.filter(profile__first_name__icontains=name) + + if request.user and not group and not request.user.is_superuser: + semi_filtered = semi_filtered & get_available_users(request.user) + + return semi_filtered + + def dehydrate_email(self, bundle): + email = "" + if bundle.request.user.is_superuser: + email = bundle.obj.email + + return email + + def dehydrate_datasets_count(self, bundle): + obj_with_perms = get_objects_for_user(bundle.request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="dataset" + ) + return ( + bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")) + .filter(metadata_only=False) + .distinct() + .count() + ) + + def dehydrate_maps_count(self, bundle): + obj_with_perms = get_objects_for_user(bundle.request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="map" + ) + return ( + bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")) + .filter(metadata_only=False) + .distinct() + .count() + ) + + def dehydrate_documents_count(self, bundle): + obj_with_perms = get_objects_for_user(bundle.request.user, "base.view_resourcebase").filter( + polymorphic_ctype__model="document" + ) + return ( + bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values("id")) + .filter(metadata_only=False) + .distinct() + .count() + ) + + def dehydrate_avatar_100(self, bundle): + return avatar_url(bundle.obj, 240) + + def dehydrate_profile_detail_url(self, bundle): + return bundle.obj.get_absolute_url() + + def dehydrate_current_user(self, bundle): + return bundle.request.user.username == bundle.obj.username + + def dehydrate_activity_stream_url(self, bundle): + return reverse( + "actstream_actor", + kwargs={"content_type_id": ContentType.objects.get_for_model(bundle.obj).pk, "object_id": bundle.obj.pk}, + ) + + def dehydrate(self, bundle): + """ + Protects user's personal information from non staff + """ + is_owner = bundle.request.user == bundle.obj + is_admin = bundle.request.user.is_staff or bundle.request.user.is_superuser + if not (is_owner or is_admin): + bundle.data = dict( + id=bundle.data.get("id", ""), + username=bundle.data.get("username", ""), + first_name=bundle.data.get("first_name", ""), + last_name=bundle.data.get("last_name", ""), + avatar_100=bundle.data.get("avatar_100", ""), + profile_detail_url=bundle.data.get("profile_detail_url", ""), + documents_count=bundle.data.get("documents_count", 0), + maps_count=bundle.data.get("maps_count", 0), + layers_count=bundle.data.get("layers_count", 0), + organization=bundle.data.get("organization", 0), + ) + return bundle + + def prepend_urls(self): + return [] + + def serialize(self, request, data, format, options=None): + if options is None: + options = {} + options["count_type"] = "owner" + + return super().serialize(request, data, format, options) + + class Meta: + queryset = get_user_model().objects.exclude(Q(username="AnonymousUser") | Q(is_active=False)) + resource_name = "profiles" + allowed_methods = ["get"] + ordering = ["username", "date_joined"] + excludes = ["is_staff", "password", "is_superuser", "is_active", "last_login"] + + filtering = { + "username": ALL, + } + serializer = CountJSONSerializer() + authorization = GeoNodePeopleAuthorization() + + +class OwnersResource(TypeFilteredResource): + """Owners api, lighter and faster version of the profiles api""" + + full_name = fields.CharField(null=True) + + def apply_filters(self, request, applicable_filters): + """filter by group if applicable by group functionality""" + + semi_filtered = super().apply_filters(request, applicable_filters) + + if request.user and not request.user.is_superuser: + semi_filtered = get_available_users(request.user) + + return semi_filtered + + def dehydrate_full_name(self, bundle): + return bundle.obj.get_full_name() or bundle.obj.username + + def dehydrate_email(self, bundle): + email = "" + if bundle.request.user.is_superuser: + email = bundle.obj.email + return email + + def dehydrate(self, bundle): + """ + Protects user's personal information from non staff + """ + is_owner = bundle.request.user == bundle.obj + is_admin = bundle.request.user.is_staff or bundle.request.user.is_superuser + if not (is_owner or is_admin): + bundle.data = dict(id=bundle.obj.id, username=bundle.obj) + return bundle + + def serialize(self, request, data, format, options=None): + if options is None: + options = {} + options["count_type"] = "owner" + + return super().serialize(request, data, format, options) + + class Meta: + queryset = get_user_model().objects.exclude(username="AnonymousUser") + resource_name = "owners" + allowed_methods = ["get"] + ordering = ["username", "date_joined"] + excludes = ["is_staff", "password", "is_superuser", "is_active", "last_login"] + + filtering = { + "username": ALL, + } + serializer = CountJSONSerializer() + authorization = ApiLockdownAuthorization() + + +class GeoserverStyleResource(ModelResource): + """Styles API for Geoserver backend.""" + + body = fields.CharField(attribute="sld_body", use_in="detail") + name = fields.CharField(attribute="name") + title = fields.CharField(attribute="sld_title") + # dataset_default_style is polymorphic, so it will have many to many + # relation + layer = fields.ManyToManyField( + "geonode.api.resourcebase_api.LayerResource", attribute="dataset_default_style", null=True + ) + version = fields.CharField(attribute="sld_version", null=True, blank=True) + style_url = fields.CharField(attribute="sld_url") + workspace = fields.CharField(attribute="workspace", null=True) + type = fields.CharField(attribute="type") + + class Meta: + paginator_class = CrossSiteXHRPaginator + queryset = Style.objects.all() + resource_name = "styles" + detail_uri_name = "id" + authorization = GeoNodeStyleAuthorization() + allowed_methods = ["get"] + filtering = {"id": ALL, "title": ALL, "name": ALL, "layer": ALL_WITH_RELATIONS} + + def build_filters(self, filters=None, **kwargs): + """Apply custom filters for layer.""" + filters = super().build_filters(filters, **kwargs) + # Convert dataset__ filters into dataset_styles__dataset__ + updated_filters = {} + for key, value in filters.items(): + key = key.replace("dataset__", "dataset_default_style__") + updated_filters[key] = value + return updated_filters + + def populate_object(self, style): + """Populate results with necessary fields + + :param style: Style objects + :type style: Style + :return: + """ + style.type = "sld" + return style + + def build_bundle(self, obj=None, data=None, request=None, **kwargs): + """Override build_bundle method to add additional info.""" + + if obj is None and self._meta.object_class: + obj = self._meta.object_class() + + elif obj: + obj = self.populate_object(obj) + + return Bundle(obj=obj, data=data, request=request, **kwargs) + + +if check_ogc_backend(geoserver.BACKEND_PACKAGE): + + class StyleResource(GeoserverStyleResource): + """Wrapper for Generic Style Resource""" + + pass + + +def _get_resource_counts(request, resourcebase_filter_kwargs): + """Return a dict with counts of resources of various types + + The ``resourcebase_filter_kwargs`` argument should be a dict with a suitable + queryset filter that can be applied to select only the relevant + ``ResourceBase`` objects to use when retrieving counts. For example:: + + _get_resource_counts( + request, + { + 'group__slug': 'my-group', + } + ) + + The above function call would result in only counting ``ResourceBase`` + objects that belong to the group that has ``my-group`` as slug + + """ + resources = get_visible_resources( + ResourceBase.objects.filter(**resourcebase_filter_kwargs), + request.user, + request=request, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + values = resources.values( + "polymorphic_ctype__model", + "is_approved", + "is_published", + ) + qs = values.annotate(counts=Count("polymorphic_ctype__model")) + types = ["dataset", "document", "map", "geoapp", "all"] + + subtypes = [] + for label, app in apps.app_configs.items(): + if hasattr(app, "type") and app.type == "GEONODE_APP": + if hasattr(app, "default_model"): + _model = apps.get_model(label, app.default_model) + if issubclass(_model, GeoApp): + types.append(_model.__name__.lower()) + subtypes.append(_model.__name__.lower()) + counts = {} + for type_ in types: + counts[type_] = { + "total": 0, + "visible": 0, + "published": 0, + "approved": 0, + } + for record in qs: + resource_type = record["polymorphic_ctype__model"] + if resource_type in subtypes: + resource_type = "geoapp" + is_visible = all((record["is_approved"], record["is_published"])) + counts["all"]["total"] += record["counts"] + counts["all"]["visible"] += record["counts"] if is_visible else 0 + counts["all"]["published"] += record["counts"] if record["is_published"] else 0 + counts["all"]["approved"] += record["counts"] if record["is_approved"] else 0 + section = counts.get(resource_type) + if section is not None: + section["total"] += record["counts"] + section["visible"] += record["counts"] if is_visible else 0 + section["published"] += record["counts"] if record["is_published"] else 0 + section["approved"] += record["counts"] if record["is_approved"] else 0 + return counts diff --git a/geonode/api/apps.py b/geonode/api/apps.py new file mode 100644 index 0000000..bf3d852 --- /dev/null +++ b/geonode/api/apps.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + + +class GeoNodeApiAppConfig(AppConfig): + name = "geonode.api" + verbose_name = "GeoNode API v1" diff --git a/geonode/api/authentication.py b/geonode/api/authentication.py new file mode 100644 index 0000000..287e3c9 --- /dev/null +++ b/geonode/api/authentication.py @@ -0,0 +1,57 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.contrib.auth.models import AnonymousUser +from oauth2_provider.models import AccessToken +from tastypie.authentication import Authentication + +from geonode.api.views import verify_access_token +from geonode.base.auth import get_token_from_auth_header + + +class OAuthAuthentication(Authentication): + def extract_auth_header(self, request): + auth_header = None + try: + auth_header = request.META.get("HTTP_AUTHORIZATION", request.META.get("HTTP_AUTHORIZATION2")) + except KeyError: + pass + return auth_header + + def token_is_valid(self, token): + valid = False + try: + verify_access_token(None, token) + valid = True + except Exception: + pass + return valid + + def is_authenticated(self, request, **kwargs): + user = AnonymousUser() + authenticated = False + if "HTTP_AUTHORIZATION" in request.META: + auth_header = self.extract_auth_header(request) + if auth_header: + access_token = get_token_from_auth_header(auth_header) + if self.token_is_valid(access_token): + obj = AccessToken.objects.get(token=access_token) + user = obj.user + authenticated = True + request.user = user + return authenticated diff --git a/geonode/api/authorization.py b/geonode/api/authorization.py new file mode 100644 index 0000000..376d53c --- /dev/null +++ b/geonode/api/authorization.py @@ -0,0 +1,185 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.db.models import Q +from tastypie.authentication import ApiKeyAuthentication +from tastypie.authorization import DjangoAuthorization +from tastypie.exceptions import Unauthorized +from tastypie.compat import get_user_model, get_username_field + +from guardian.shortcuts import get_objects_for_user +from tastypie.http import HttpUnauthorized + +from django.conf import settings + +from geonode import geoserver +from geonode.utils import check_ogc_backend + + +class GeoNodeAuthorization(DjangoAuthorization): + """Object level API authorization based on GeoNode granular + permission system""" + + def read_list(self, object_list, bundle): + permitted_ids = [] + try: + permitted_ids = get_objects_for_user(bundle.request.user, "base.view_resourcebase").values("id") + except Exception: + pass + + return object_list.filter(id__in=permitted_ids) + + def read_detail(self, object_list, bundle): + if "schema" in bundle.request.path: + return True + return bundle.request.user.has_perm("view_resourcebase", bundle.obj.get_self_resource()) + + def create_list(self, object_list, bundle): + # TODO implement if needed + raise Unauthorized() + + def create_detail(self, object_list, bundle): + return bundle.request.user.has_perm("add_resourcebase", bundle.obj.get_self_resource()) + + def update_list(self, object_list, bundle): + # TODO implement if needed + raise Unauthorized() + + def update_detail(self, object_list, bundle): + return bundle.request.user.has_perm("change_resourcebase", bundle.obj.get_self_resource()) + + def delete_list(self, object_list, bundle): + # TODO implement if needed + raise Unauthorized() + + def delete_detail(self, object_list, bundle): + return bundle.request.user.has_perm("delete_resourcebase", bundle.obj.get_self_resource()) + + +class GeonodeApiKeyAuthentication(ApiKeyAuthentication): + """ + Override ApiKeyAuthentication to prevent 401 response when no api key is provided. + """ + + def is_authenticated(self, request, **kwargs): + """ + Finds the user and checks their API key. + + Should return either ``True`` if allowed, ``False`` if not or an + ``HttpResponse`` if you need something custom. + """ + + try: + username, api_key = self.extract_credentials(request) + except ValueError: + return self._unauthorized() + + if not username or not api_key: + return True + + username_field = get_username_field() + User = get_user_model() + + try: + lookup_kwargs = {username_field: username} + user = User.objects.get(**lookup_kwargs) + except (User.DoesNotExist, User.MultipleObjectsReturned): + return self._unauthorized() + + if not self.check_active(user): + return False + + key_auth_check = self.get_key(user, api_key) + if key_auth_check and not isinstance(key_auth_check, HttpUnauthorized): + request.user = user + + return key_auth_check + + +class GeoNodeStyleAuthorization(GeoNodeAuthorization): + """Object level API authorization based on GeoNode granular + permission system + + Style object permissions should follow it's layer permissions + """ + + def filter_by_resource_ids(self, object_list, permitted_ids): + """Filter Style queryset by permitted resource ids.""" + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + return object_list.filter(dataset_styles__id__in=permitted_ids) + + def read_list(self, object_list, bundle): + permitted_ids = get_objects_for_user(bundle.request.user, "base.view_resourcebase").values("id") + + return self.filter_by_resource_ids(object_list, permitted_ids) + + def delete_detail(self, object_list, bundle): + permitted_ids = get_objects_for_user(bundle.request.user, "layer.change_dataset_style").values("id") + + resource_obj = bundle.obj.get_self_resource() + return resource_obj in permitted_ids + + +class ApiLockdownAuthorization(DjangoAuthorization): + """API authorization for all resources which are not protected by others authentication/authorization mechanism. + If setting "API_LOCKDOWN" is set to True, resource can only be accessed by authenticated users. For anonymous + requests, empty lists are returned. + """ + + def read_list(self, object_list, bundle): + user = bundle.request.user + if settings.API_LOCKDOWN and not user.is_authenticated: + # return empty list + return [] + else: + return object_list + + +class GeoNodePeopleAuthorization(DjangoAuthorization): + """API authorization that allows only authenticated users to view list of users""" + + def read_list(self, object_list, bundle): + user = bundle.request.user + if not user.is_authenticated: + # return empty list + return [] + return object_list + + +class GroupAuthorization(ApiLockdownAuthorization): + def read_list(self, object_list, bundle): + groups = super().read_list(object_list, bundle) + user = bundle.request.user + if groups: + if not user.is_authenticated or user.is_anonymous: + return groups.exclude(groupprofile__access="private") + elif not user.is_superuser: + return groups.filter(Q(groupprofile__in=user.group_list_all()) | ~Q(groupprofile__access="private")) + return groups + + +class GroupProfileAuthorization(ApiLockdownAuthorization): + def read_list(self, object_list, bundle): + groups = super().read_list(object_list, bundle) + user = bundle.request.user + if groups: + if not user.is_authenticated or user.is_anonymous: + return groups.exclude(access="private") + elif not user.is_superuser: + return groups.filter(Q(pk__in=user.group_list_all()) | ~Q(access="private")) + return groups diff --git a/geonode/api/paginator.py b/geonode/api/paginator.py new file mode 100644 index 0000000..7f2f792 --- /dev/null +++ b/geonode/api/paginator.py @@ -0,0 +1,80 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.conf import settings +from tastypie.exceptions import BadRequest +from tastypie.paginator import Paginator + + +class CrossSiteXHRPaginator(Paginator): + def get_limit(self): + """ + Determines the proper maximum number of results to return. + + In order of importance, it will use: + + * The user-requested ``limit`` from the GET parameters, if specified. + * The object-level ``limit`` if specified. + * ``settings.API_LIMIT_PER_PAGE`` if specified. + + Default is 20 per page. + """ + + limit = self.request_data.get("limit", self.limit) + if limit is None: + limit = getattr(settings, "API_LIMIT_PER_PAGE", 20) + + try: + limit = int(limit) + except ValueError: + raise BadRequest("Invalid limit provided. Please provide a positive integer.") + + if limit < 0: + raise BadRequest("Invalid limit provided. Please provide a positive integer >= 0.") + + if self.max_limit and (not limit or limit > self.max_limit): + # If it's more than the max, we're only going to return the max. + # This is to prevent excessive DB (or other) load. + return self.max_limit + + return limit + + def get_offset(self): + """ + Determines the proper starting offset of results to return. + + It attempts to use the user-provided ``offset`` from the GET parameters, + if specified. Otherwise, it falls back to the object-level ``offset``. + + Default is 0. + """ + offset = self.offset + + if "offset" in self.request_data: + offset = self.request_data["offset"] + + try: + offset = int(offset) + except ValueError: + raise BadRequest("Invalid offset provided. Please provide an integer.") + + if offset < 0: + raise BadRequest("Invalid offset provided. Please provide a positive integer >= 0.") + + return offset diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py new file mode 100644 index 0000000..cf98190 --- /dev/null +++ b/geonode/api/resourcebase_api.py @@ -0,0 +1,673 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.base.enumerations import LAYER_TYPES +import logging + +from django.db.models import Q +from django.http import HttpResponse +from django.conf import settings +from tastypie.authentication import MultiAuthentication, SessionAuthentication +from tastypie.bundle import Bundle + +from tastypie.constants import ALL, ALL_WITH_RELATIONS +from tastypie.resources import ModelResource +from tastypie import fields + +from django.core.exceptions import ObjectDoesNotExist +from django.forms.models import model_to_dict + +from tastypie.utils.mime import build_content_type + +from geonode import get_version, geoserver +from geonode.layers.models import Dataset +from geonode.maps.models import Map +from geonode.geoapps.models import GeoApp +from geonode.documents.models import Document +from geonode.base.models import ResourceBase +from geonode.base.models import HierarchicalKeyword +from geonode.base.bbox_utils import filter_bbox +from geonode.groups.models import GroupProfile +from geonode.utils import check_ogc_backend +from geonode.security.utils import get_visible_resources +from .authentication import OAuthAuthentication +from .authorization import GeoNodeAuthorization, GeonodeApiKeyAuthentication + +from .api import ( + TagResource, + RegionResource, + OwnersResource, + ThesaurusKeywordResource, + TopicCategoryResource, + GroupResource, + FILTER_TYPES, +) +from .paginator import CrossSiteXHRPaginator +from django.utils.translation import gettext as _ + +logger = logging.getLogger(__name__) + + +class CommonMetaApi: + authorization = GeoNodeAuthorization() + allowed_methods = ["get"] + filtering = { + "title": ALL, + "keywords": ALL_WITH_RELATIONS, + "tkeywords": ALL_WITH_RELATIONS, + "regions": ALL_WITH_RELATIONS, + "category": ALL_WITH_RELATIONS, + "group": ALL_WITH_RELATIONS, + "owner": ALL_WITH_RELATIONS, + "date": ALL, + "purpose": ALL, + "uuid": ALL_WITH_RELATIONS, + "abstract": ALL, + "metadata": ALL_WITH_RELATIONS, + } + ordering = ["date", "title", "popular_count"] + max_limit = None + + +class CommonModelApi(ModelResource): + keywords = fields.ToManyField(TagResource, "keywords", null=True) + regions = fields.ToManyField(RegionResource, "regions", null=True) + category = fields.ToOneField(TopicCategoryResource, "category", null=True, full=True) + group = fields.ToOneField(GroupResource, "group", null=True, full=True) + owner = fields.ToOneField(OwnersResource, "owner", full=True) + tkeywords = fields.ToManyField(ThesaurusKeywordResource, "tkeywords", null=True) + VALUES = [ + # fields in the db + "id", + "uuid", + "name", + "typename", + "title", + "date", + "date_type", + "edition", + "purpose", + "maintenance_frequency", + "restriction_code_type", + "constraints_other", + "license", + "language", + "spatial_representation_type", + "temporal_extent_start", + "temporal_extent_end", + "data_quality_statement", + "abstract", + "csw_wkt_geometry", + "csw_type", + "owner__username", + "share_count", + "popular_count", + "srid", + "bbox_polygon", + "category__gn_description", + "supplemental_information", + "site_url", + "thumbnail_url", + "detail_url", + "rating", + "group__name", + "has_time", + "is_approved", + "is_published", + "dirty_state", + "metadata_only", + ] + + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): + if filters is None: + filters = {} + orm_filters = super().build_filters(filters=filters, ignore_bad_filters=ignore_bad_filters, **kwargs) + if "type__in" in filters and (filters["type__in"] in FILTER_TYPES.keys() or filters["type__in"] in LAYER_TYPES): + orm_filters.update({"type": filters.getlist("type__in")}) + if "app_type__in" in filters: + orm_filters.update({"resource_type": filters["app_type__in"].lower()}) + + _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith("metadata__")} + if _metadata: + orm_filters.update({"metadata_filters": _metadata}) + + if "extent" in filters: + orm_filters.update({"extent": filters["extent"]}) + orm_filters["f_method"] = filters["f_method"] if "f_method" in filters else "and" + if not settings.SEARCH_RESOURCES_EXTENDED: + return self._remove_additional_filters(orm_filters) + return orm_filters + + def _remove_additional_filters(self, orm_filters): + orm_filters.pop("abstract__icontains", None) + orm_filters.pop("purpose__icontains", None) + orm_filters.pop("f_method", None) + return orm_filters + + def apply_filters(self, request, applicable_filters): + types = applicable_filters.pop("type", None) + extent = applicable_filters.pop("extent", None) + keywords = applicable_filters.pop("keywords__slug__in", None) + metadata_only = applicable_filters.pop("metadata_only", False) + filtering_method = applicable_filters.pop("f_method", "and") + metadata_filters = applicable_filters.pop("metadata_filters", None) + if filtering_method == "or": + filters = Q() + for f in applicable_filters.items(): + filters |= Q(f) + semi_filtered = self.get_object_list(request).filter(filters) + else: + semi_filtered = super().apply_filters(request, applicable_filters) + filtered = None + if types: + for the_type in types: + if the_type in LAYER_TYPES: + super_type = the_type + if "vector_time" == the_type: + super_type = "vector" + if filtered: + if "time" in the_type: + filtered = filtered | semi_filtered.filter(Layer___subtype=super_type).exclude( + Layer___has_time=False + ) + else: + filtered = filtered | semi_filtered.filter(Layer___subtype=super_type) + else: + if "time" in the_type: + filtered = semi_filtered.filter(Layer___subtype=super_type).exclude(Layer___has_time=False) + else: + filtered = semi_filtered.filter(Layer___subtype=super_type) + else: + _type_filter = FILTER_TYPES[the_type].__name__.lower() + if filtered: + filtered = filtered | semi_filtered.filter(polymorphic_ctype__model=_type_filter) + else: + filtered = semi_filtered.filter(polymorphic_ctype__model=_type_filter) + else: + filtered = semi_filtered + + if extent: + filtered = filter_bbox(filtered, extent) + + if keywords: + filtered = self.filter_h_keywords(filtered, keywords) + + if metadata_filters: + filtered = filtered.filter(**metadata_filters) + + # return filtered + return get_visible_resources( + filtered, + request.user if request else None, + metadata_only=metadata_only, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + def filter_h_keywords(self, queryset, keywords): + treeqs = HierarchicalKeyword.objects.none() + if keywords and len(keywords) > 0: + for keyword in keywords: + try: + kws = HierarchicalKeyword.objects.filter(Q(name__iexact=keyword) | Q(slug__iexact=keyword)) + for kw in kws: + treeqs = treeqs | HierarchicalKeyword.get_tree(kw) + except ObjectDoesNotExist: + # Ignore keywords not actually used? + pass + filtered = queryset.filter(Q(keywords__in=treeqs)) + else: + filtered = queryset + return filtered + + def get_list(self, request, **kwargs): + """ + Returns a serialized list of resources. + + Calls ``obj_get_list`` to provide the data, then handles that result + set and serializes it. + + Should return a HttpResponse (200 OK). + """ + # TODO: Uncached for now. Invalidation that works for everyone may be + # impossible. + base_bundle = self.build_bundle(request=request) + objects = self.obj_get_list(bundle=base_bundle, **self.remove_api_resource_names(kwargs)) + sorted_objects = self.apply_sorting(objects, options=request.GET) + + paginator = self._meta.paginator_class( + request.GET, + sorted_objects, + resource_uri=self.get_resource_uri(), + limit=self._meta.limit, + max_limit=self._meta.max_limit, + collection_name=self._meta.collection_name, + ) + to_be_serialized = paginator.page() + + to_be_serialized = self.alter_list_data_to_serialize(request, to_be_serialized) + + return self.create_response(request, to_be_serialized, response_objects=objects) + + def format_objects(self, objects): + """ + Format the objects for output in a response. + """ + for key in ("site_url", "has_time"): + if key in self.VALUES: + idx = self.VALUES.index(key) + del self.VALUES[idx] + + # hack needed because dehydrate does not seem to work in CommonModelApi + formatted_objects = [] + for obj in objects: + formatted_obj = model_to_dict(obj, fields=self.VALUES) + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL + + formatted_obj["owner__username"] = obj.owner.username + formatted_obj["owner_name"] = obj.owner.get_full_name() or obj.owner.username + + if formatted_obj.get("metadata", None): + formatted_obj["metadata"] = [model_to_dict(_m) for _m in formatted_obj["metadata"]] + + formatted_obj["detail_url"] = obj.detail_url + + formatted_objects.append(formatted_obj) + + return formatted_objects + + def create_response(self, request, data, response_class=HttpResponse, response_objects=None, **response_kwargs): + """ + Extracts the common "which-format/serialize/return-response" cycle. + + Mostly a useful shortcut/hook. + """ + + # If an user does not have at least view permissions, he won't be able + # to see the resource at all. + filtered_objects_ids = None + try: + if data["objects"]: + filtered_objects_ids = [ + item.id + for item in data["objects"] + if request.user.has_perm("view_resourcebase", item.get_self_resource()) + ] + except Exception: + pass + + if isinstance(data, dict) and "objects" in data and not isinstance(data["objects"], list): + if filtered_objects_ids: + data["objects"] = [ + x for x in list(self.format_objects(data["objects"])) if x["id"] in filtered_objects_ids + ] + else: + data["objects"] = list(self.format_objects(data["objects"])) + + # give geonode version + data["geonode_version"] = get_version() + + desired_format = self.determine_format(request) + serialized = self.serialize(request, data, desired_format) + + return response_class(content=serialized, content_type=build_content_type(desired_format), **response_kwargs) + + def prepend_urls(self): + return [] + + def hydrate_title(self, bundle): + title = bundle.data.get("title", None) + if title: + bundle.data["title"] = title.replace(",", "_") + return bundle + + +class ResourceBaseResource(CommonModelApi): + """ResourceBase api""" + + class Meta(CommonMetaApi): + paginator_class = CrossSiteXHRPaginator + queryset = ResourceBase.objects.polymorphic_queryset().distinct().order_by("-date") + resource_name = "base" + excludes = ["csw_anytext", "metadata_xml"] + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) + + +class FeaturedResourceBaseResource(CommonModelApi): + """Only the featured resourcebases""" + + class Meta(CommonMetaApi): + paginator_class = CrossSiteXHRPaginator + queryset = ResourceBase.objects.filter(featured=True).order_by("-date") + resource_name = "featured" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) + + +class LayerResource(CommonModelApi): + """Dataset API""" + + links = fields.ListField(attribute="links", null=True, use_in="all", default=[]) + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + default_style = fields.ForeignKey("geonode.api.api.StyleResource", attribute="default_style", null=True) + styles = fields.ManyToManyField("geonode.api.api.StyleResource", attribute="styles", null=True, use_in="detail") + + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): + _filters = filters.copy() + metadata_only = _filters.pop("metadata_only", False) + orm_filters = super().build_filters(_filters) + orm_filters["metadata_only"] = False if not metadata_only else metadata_only[0] + return orm_filters + + def format_objects(self, objects): + """ + Formats the object. + """ + formatted_objects = [] + for obj in objects: + # convert the object to a dict using the standard values. + # includes other values + values = self.VALUES + ["alternate", "name"] + formatted_obj = model_to_dict(obj, fields=values) + username = obj.owner.get_username() + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name + if obj.category: + formatted_obj["category__gn_description"] = _(obj.category.gn_description) + if obj.group: + formatted_obj["group"] = obj.group + try: + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) + except GroupProfile.DoesNotExist: + formatted_obj["group_name"] = obj.group + + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] + + # provide style information + bundle = self.build_bundle(obj=obj) + formatted_obj["default_style"] = self.default_style.dehydrate(bundle, for_list=True) + + # Add resource uri + formatted_obj["resource_uri"] = self.get_resource_uri(bundle) + + formatted_obj["links"] = self.dehydrate_ogc_links(bundle) + + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL + + # Probe Remote Services + formatted_obj["store_type"] = "dataset" + formatted_obj["online"] = True + if hasattr(obj, "subtype"): + formatted_obj["store_type"] = obj.subtype + if obj.subtype in ["tileStore", "remote"] and hasattr(obj, "remote_service"): + if obj.remote_service: + formatted_obj["online"] = obj.remote_service.probe == 200 + else: + formatted_obj["online"] = False + + formatted_obj["gtype"] = self.dehydrate_gtype(bundle) + + formatted_obj["processed"] = obj.instance_is_processed + # put the object on the response stack + formatted_objects.append(formatted_obj) + return formatted_objects + + def _dehydrate_links(self, bundle, link_types=None): + """Dehydrate links field.""" + + dehydrated = [] + obj = bundle.obj + link_fields = ["extension", "link_type", "name", "mime", "url"] + + links = obj.link_set.all() + if link_types: + links = links.filter(link_type__in=link_types) + for lnk in links: + formatted_link = model_to_dict(lnk, fields=link_fields) + dehydrated.append(formatted_link) + + return dehydrated + + def dehydrate_links(self, bundle): + return self._dehydrate_links(bundle) + + def dehydrate_ogc_links(self, bundle): + return self._dehydrate_links(bundle, ["OGC:WMS", "OGC:WFS", "OGC:WCS"]) + + def dehydrate_gtype(self, bundle): + return bundle.obj.gtype + + def build_bundle(self, obj=None, data=None, request=None, **kwargs): + """Override build_bundle method to add additional info.""" + + if obj is None and self._meta.object_class: + obj = self._meta.object_class() + elif obj: + obj = self.populate_object(obj) + + return Bundle(obj=obj, data=data, request=request, **kwargs) + + def populate_object(self, obj): + """Populate results with necessary fields + + :param obj: Dataset obj + :type obj: Dataset + :return: + """ + return obj + + # copy parent attribute before modifying + VALUES = CommonModelApi.VALUES[:] + VALUES.append("typename") + + class Meta(CommonMetaApi): + paginator_class = CrossSiteXHRPaginator + queryset = Dataset.objects.distinct().order_by("-date") + resource_name = "datasets" + detail_uri_name = "id" + include_resource_uri = True + allowed_methods = ["get", "patch"] + excludes = ["csw_anytext", "metadata_xml"] + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) + filtering = CommonMetaApi.filtering + # Allow filtering using ID + filtering.update({"id": ALL, "name": ALL, "alternate": ALL, "metadata_only": ALL}) + + +class MapResource(CommonModelApi): + """Maps API""" + + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): + _filters = filters.copy() + metadata_only = _filters.pop("metadata_only", False) + orm_filters = super().build_filters(_filters) + orm_filters["metadata_only"] = False if not metadata_only else metadata_only[0] + return orm_filters + + def format_objects(self, objects): + """ + Formats the objects and provides reference to list of layers in map + resources. + + :param objects: Map objects + """ + formatted_objects = [] + for obj in objects: + # convert the object to a dict using the standard values. + formatted_obj = model_to_dict(obj, fields=self.VALUES) + username = obj.owner.get_username() + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name + if obj.category: + formatted_obj["category__gn_description"] = _(obj.category.gn_description) + if obj.group: + formatted_obj["group"] = obj.group + try: + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) + except GroupProfile.DoesNotExist: + formatted_obj["group_name"] = obj.group + + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] + + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL + + # Probe Remote Services + formatted_obj["store_type"] = "map" + formatted_obj["online"] = True + + # get map layers + map_datasets = obj.maplayers + formatted_datasets = [] + map_dataset_fields = ["id", "name", "ows_url", "local"] + for layer in map_datasets.iterator(): + formatted_map_dataset = model_to_dict(layer, fields=map_dataset_fields) + formatted_datasets.append(formatted_map_dataset) + formatted_obj["layers"] = formatted_datasets + + formatted_objects.append(formatted_obj) + return formatted_objects + + class Meta(CommonMetaApi): + paginator_class = CrossSiteXHRPaginator + queryset = Map.objects.distinct().order_by("-date") + resource_name = "maps" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) + + +class GeoAppResource(CommonModelApi): + """GeoApps API""" + + def format_objects(self, objects): + """ + Formats the objects and provides reference to list of layers in GeoApp + resources. + + :param objects: GeoApp objects + """ + formatted_objects = [] + for obj in objects: + # convert the object to a dict using the standard values. + formatted_obj = model_to_dict(obj, fields=self.VALUES) + username = obj.owner.get_username() + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name + if obj.category: + formatted_obj["category__gn_description"] = obj.category.gn_description + if obj.group: + formatted_obj["group"] = obj.group + try: + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) + except GroupProfile.DoesNotExist: + formatted_obj["group_name"] = obj.group + + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] + + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL + + # Probe Remote Services + formatted_obj["store_type"] = "geoapp" + formatted_obj["online"] = True + + formatted_objects.append(formatted_obj) + return formatted_objects + + class Meta(CommonMetaApi): + paginator_class = CrossSiteXHRPaginator + filtering = CommonMetaApi.filtering + filtering.update({"app_type": ALL}) + queryset = GeoApp.objects.distinct().order_by("-date") + resource_name = "geoapps" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) + + +class DocumentResource(CommonModelApi): + """Documents API""" + + def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): + _filters = filters.copy() + metadata_only = _filters.pop("metadata_only", False) + orm_filters = super().build_filters(_filters) + orm_filters["metadata_only"] = False if not metadata_only else metadata_only[0] + return orm_filters + + def format_objects(self, objects): + """ + Formats the objects and provides reference to list of layers in map + resources. + + :param objects: Map objects + """ + formatted_objects = [] + for obj in objects: + # convert the object to a dict using the standard values. + formatted_obj = model_to_dict(obj, fields=self.VALUES) + username = obj.owner.get_username() + full_name = obj.owner.get_full_name() or username + formatted_obj["owner__username"] = username + formatted_obj["owner_name"] = full_name + if obj.category: + formatted_obj["category__gn_description"] = _(obj.category.gn_description) + if obj.group: + formatted_obj["group"] = obj.group + try: + formatted_obj["group_name"] = GroupProfile.objects.get(slug=obj.group.name) + except GroupProfile.DoesNotExist: + formatted_obj["group_name"] = obj.group + + formatted_obj["keywords"] = [k.name for k in obj.keywords.all()] if obj.keywords else [] + formatted_obj["regions"] = [r.name for r in obj.regions.all()] if obj.regions else [] + + if "site_url" not in formatted_obj or len(formatted_obj["site_url"]) == 0: + formatted_obj["site_url"] = settings.SITEURL + + # Probe Remote Services + formatted_obj["store_type"] = "dataset" + formatted_obj["online"] = True + + formatted_objects.append(formatted_obj) + return formatted_objects + + class Meta(CommonMetaApi): + paginator_class = CrossSiteXHRPaginator + filtering = CommonMetaApi.filtering + filtering.update({"subtype": ALL}) + queryset = Document.objects.distinct().order_by("-date") + resource_name = "documents" + authentication = MultiAuthentication( + SessionAuthentication(), OAuthAuthentication(), GeonodeApiKeyAuthentication() + ) diff --git a/geonode/api/tests.py b/geonode/api/tests.py new file mode 100644 index 0000000..95692f2 --- /dev/null +++ b/geonode/api/tests.py @@ -0,0 +1,1029 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +from datetime import datetime, timedelta +from tastypie.test import ResourceTestCaseMixin, TestApiClient +from unittest.mock import patch +from urllib.parse import urlencode +from uuid import uuid4 + +from django.conf import settings +from django.urls import reverse +from django.contrib.auth.models import Group +from django.contrib.auth import get_user_model +from django.test.utils import override_settings + +from guardian.shortcuts import get_anonymous_user + +from geonode import geoserver +from geonode.geoserver.manager import GeoServerResourceManager +from geonode.maps.models import Map +from geonode.layers.models import Dataset +from geonode.documents.models import Document +from geonode.base.models import ( + ExtraMetadata, + Thesaurus, + ThesaurusLabel, + ThesaurusKeyword, + ThesaurusKeywordLabel, + ResourceBase, +) +from geonode.utils import check_ogc_backend +from geonode.decorators import on_ogc_backend +from geonode.groups.models import GroupProfile +from geonode.base.auth import get_or_create_token +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.base.populate_test_data import all_public, create_models, remove_models + + +logger = logging.getLogger(__name__) + + +class UserAndTokenInfoApiTests(GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def test_userinfo_response(self): + userinfo_url = reverse("userinfo") + _user = get_user_model().objects.get(username="bobby") + self.client.login(username="bobby", password="bob") + response = self.client.get(userinfo_url) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["sub"], str(_user.pk)) + self.client.logout() + response = self.client.get(userinfo_url) + self.assertEqual(response.status_code, 401) + + def test_tokeninfo_response(self): + tokeninfo_url = reverse("tokeninfo") + _user = get_user_model().objects.get(username="bobby") + token = get_or_create_token(_user) + response = self.client.post(tokeninfo_url, data={"token": token}) + self.assertEqual(response.status_code, 200) + response_json = response.json() + self.assertEqual(response_json["access_token"], token.token) + self.assertEqual(response_json["user_id"], _user.pk) + + +class PermissionsApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.user = "admin" + self.passwd = "admin" + self.perm_spec = {"users": {}, "groups": {}} + + def test_dataset_get_list_unauth_all_public(self): + """ + Test that the correct number of layers are returned when the + client is not logged in and all are public + """ + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) + + def test_datasets_get_list_unauth_some_public(self): + """ + Test that if a layer is not public then not all are returned when the + client is not logged in + """ + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + layer = Dataset.objects.first() + layer.set_permissions(self.perm_spec) + + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) + + def test_datasets_get_list_auth_some_public(self): + """ + Test that if a layer is not public then all are returned if the + client is not logged in + """ + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + self.api_client.client.login(username=self.user, password=self.passwd) + layer = Dataset.objects.first() + layer.set_permissions(self.perm_spec) + + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) + + def test_dataset_get_list_dataset_private_to_one_user(self): + """ + Test that if a layer is only visible by admin, then does not appear + in the unauthenticated list nor in the list when logged is as bobby + """ + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + perm_spec = {"users": {"admin": ["view_resourcebase"]}, "groups": {}} + layer = Dataset.objects.first() + layer.set_permissions(perm_spec) + resp = self.api_client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) + + self.api_client.client.login(username="bobby", password="bob") + resp = self.api_client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) + + self.api_client.client.login(username=self.user, password=self.passwd) + resp = self.api_client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) + + layer.is_published = False + layer.save() + + # with resource publishing + with self.settings(RESOURCE_PUBLISHING=True): + resp = self.api_client.get(list_url) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + self.api_client.client.login(username="bobby", password="bob") + resp = self.api_client.get(list_url) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + self.api_client.client.login(username=self.user, password=self.passwd) + resp = self.api_client.get(list_url) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + def test_dataset_get_detail_unauth_dataset_not_public(self): + """ + Test that layer detail gives 404 when not public and not logged in + """ + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + resp = self.client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) + + layer = Dataset.objects.first() + layer.set_permissions(self.perm_spec) + layer.clear_dirty_state() + + resp = self.client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) + self.assertHttpNotFound(self.client.get(f"{list_url + str(layer.id)}/")) + + self.client.login(username=self.user, password=self.passwd) + self.assertValidJSONResponse(self.client.get(f"{list_url + str(layer.id)}/")) + + # with delayed security + with self.settings(DELAYED_SECURITY_SIGNALS=True, GEOFENCE_SECURITY_ENABLED=True): + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + gm = GeoServerResourceManager() + gm.set_permissions(layer.uuid, instance=layer, permissions=self.perm_spec) + self.assertTrue(layer.dirty_state) + + self.client.login(username=self.user, password=self.passwd) + resp = self.client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) # admin can't see resources in dirty_state + + self.client.logout() + resp = self.client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) + + from django.contrib.auth import get_user_model + + get_user_model().objects.create( + username="imnew", + password="pbkdf2_sha256$12000$UE4gAxckVj4Z$N6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=", + ) + self.client.login(username="imnew", password="thepwd") + resp = self.client.get(list_url) + self.assertEqual(len(self.deserialize(resp)["objects"]), 7) + + def test_new_user_has_access_to_old_datasets(self): + """Test that a new user can access the public available layers""" + from django.contrib.auth import get_user_model + + get_user_model().objects.create( + username="imnew", + password="pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ + 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=", + ) + + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + self.api_client.client.login(username="imnew", password="thepwd") + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + # with delayed security + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + _ogc_geofence_enabled = settings.OGC_SERVER + try: + _ogc_geofence_enabled["default"]["GEOFENCE_SECURITY_ENABLED"] = True + with self.settings( + DELAYED_SECURITY_SIGNALS=True, + OGC_SERVER=_ogc_geofence_enabled, + DEFAULT_ANONYMOUS_VIEW_PERMISSION=True, + ): + layer = Dataset.objects.first() + layer.set_default_permissions() + layer.refresh_from_db() + # self.assertTrue(layer.dirty_state) + + self.client.login(username=self.user, password=self.passwd) + resp = self.client.get(list_url) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + self.client.logout() + resp = self.client.get(list_url) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + self.client.login(username="imnew", password="thepwd") + resp = self.client.get(list_url) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + finally: + _ogc_geofence_enabled["default"]["GEOFENCE_SECURITY_ENABLED"] = False + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_outh_token(self): + user = "admin" + _user = get_user_model().objects.get(username=user) + token = get_or_create_token(_user) + auth_header = f"Bearer {token}" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + with self.settings(SESSION_EXPIRED_CONTROL_ENABLED=False, DELAYED_SECURITY_SIGNALS=False): + # all public + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + perm_spec = {"users": {"admin": ["view_resourcebase"]}, "groups": {}} + layer = Dataset.objects.first() + layer.set_permissions(perm_spec) + resp = self.api_client.get(list_url) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + resp = self.api_client.get(list_url, authentication=auth_header) + self.assertGreaterEqual(len(self.deserialize(resp)["objects"]), 7) + + layer.is_published = False + layer.save() + + @override_settings(API_LOCKDOWN=True) + def test_api_lockdown_false(self): + profiles_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "profiles"}) + + # test if results are returned for anonymous users if API_LOCKDOWN is set to False in settings + filter_url = profiles_list_url + + with self.settings(API_LOCKDOWN=False): + # anonymous + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + # admin + self.api_client.client.login(username="admin", password="admin") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 9) + + @override_settings(API_LOCKDOWN=True) + def test_profiles_lockdown(self): + profiles_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "profiles"}) + + filter_url = profiles_list_url + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + # now test with logged in user + self.api_client.client.login(username="bobby", password="bob") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 6) + # Returns limitted info about other users + bobby = get_user_model().objects.get(username="bobby") + profiles = self.deserialize(resp)["objects"] + for profile in profiles: + if profile["username"] == "bobby": + self.assertEquals(profile.get("email"), bobby.email) + else: + self.assertIsNone(profile.get("email")) + + @override_settings(API_LOCKDOWN=True) + def test_owners_lockdown(self): + owners_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "owners"}) + + filter_url = owners_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + # now test with logged in user + self.api_client.client.login(username="bobby", password="bob") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 6) + # Returns limitted info about other users + bobby = get_user_model().objects.get(username="bobby") + owners = self.deserialize(resp)["objects"] + for owner in owners: + if owner["username"] == "bobby": + self.assertEquals(owner.get("email"), bobby.email) + else: + self.assertIsNone(owner.get("email")) + self.assertIsNone(owner.get("first_name")) + + # now test with logged in admin + self.api_client.client.login(username="admin", password="admin") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 9) + + @override_settings(API_LOCKDOWN=True) + def test_groups_lockdown(self): + groups_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "groups"}) + + filter_url = groups_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + # now test with logged in user + self.api_client.client.login(username="bobby", password="bob") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + @override_settings(API_LOCKDOWN=True) + def test_regions_lockdown(self): + region_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "regions"}) + + filter_url = region_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + self.api_client.client.login(username="bobby", password="bob") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertTrue(len(self.deserialize(resp)["objects"]) >= 200) + + @override_settings(API_LOCKDOWN=True) + def test_tags_lockdown(self): + tag_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "keywords"}) + + filter_url = tag_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + self.api_client.client.login(username="bobby", password="bob") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 5) + + +class SearchApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + """Test the search""" + + # loading test thesausuri and initial data + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json", "test_thesaurus.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + + self.norman = get_user_model().objects.get(username="norman") + self.norman.groups.add(Group.objects.get(name="anonymous")) + self.test_user = get_user_model().objects.get(username="test_user") + self.test_user.groups.add(Group.objects.get(name="anonymous")) + self.bar = GroupProfile.objects.get(slug="bar") + self.anonymous_user = get_anonymous_user() + self.profiles_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "profiles"}) + self.groups_list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "groups"}) + + def test_profiles_filters(self): + """Test profiles filtering""" + + with self.settings(API_LOCKDOWN=False): + filter_url = self.profiles_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + filter_url = f"{self.profiles_list_url}?name__icontains=norm" + # Anonymous + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + self.api_client.client.login(username="admin", password="admin") + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{self.profiles_list_url}?name__icontains=NoRmAN" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{self.profiles_list_url}?name__icontains=bar" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + def test_groups_filters(self): + """Test groups filtering""" + + with self.settings(API_LOCKDOWN=False): + filter_url = self.groups_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{self.groups_list_url}?name__icontains=bar" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{self.groups_list_url}?name__icontains=BaR" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{self.groups_list_url}?name__icontains=foo" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + def test_category_filters(self): + """Test category filtering""" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + # check we get the correct layers number returnered filtering on one + # and then two different categories + filter_url = f"{list_url}?category__identifier=location" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 3) + + filter_url = f"{list_url}?category__identifier__in=location&category__identifier__in=biota" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 5) + + def test_metadata_filters(self): + """Test category filtering""" + _r = Dataset.objects.first() + _m = ExtraMetadata.objects.create( + resource=_r, + metadata={ + "name": "metadata-updated", + "slug": "metadata-slug-updated", + "help_text": "this is the help text-updated", + "field_type": "str-updated", + "value": "my value-updated", + "category": "category", + }, + ) + + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + _r.metadata.add(_m) + # check we get the correct layers number returnered filtering on one + # and then two different categories + filter_url = f"{list_url}?metadata__category=category" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{list_url}?metadata__category=not-existing-category" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + def test_tag_filters(self): + """Test keywords filtering""" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + # check we get the correct layers number returnered filtering on one + # and then two different keywords + filter_url = f"{list_url}?keywords__slug=layertagunique" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{list_url}?keywords__slug__in=layertagunique&keywords__slug__in=populartag" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 8) + + def test_owner_filters(self): + """Test owner filtering""" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + # check we get the correct layers number returnered filtering on one + # and then two different owners + filter_url = f"{list_url}?owner__username=user1" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + filter_url = f"{list_url}?owner__username__in=user1&owner__username__in=foo" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 2) + + def test_title_filter(self): + """Test title filtering""" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + # check we get the correct layers number returnered filtering on the + # title + filter_url = f"{list_url}?title=layer2" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + def test_date_filter(self): + """Test date filtering""" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + # check we get the correct layers number returnered filtering on the + # dates + step = timedelta(days=60) + now = datetime.now() + fstring = "%Y-%m-%d" + + def to_date(val): + return val.date().strftime(fstring) + + d1 = to_date(now - step) + filter_url = f"{list_url}?date__exact={d1}" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 0) + + d3 = to_date(now - (3 * step)) + filter_url = f"{list_url}?date__gte={d3}" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 3) + + d4 = to_date(now - (4 * step)) + filter_url = f"{list_url}?date__range={d4},{to_date(now)}" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 4) + + def test_extended_text_filter(self): + """Test that the extended text filter works as expected""" + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + + filter_url = ( + f"{list_url}?title__icontains=layer2&abstract__icontains=layer2&purpose__icontains=layer2&f_method=or" + ) + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)["objects"]), 1) + + def test_the_api_should_return_all_datasets_with_metadata_false(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + user = get_user_model().objects.get(username="admin") + token = get_or_create_token(user) + auth_header = f"Bearer {token}" + + resp = self.api_client.get(list_url, authentication=auth_header) + self.assertValidJSONResponse(resp) + self.assertEqual(Dataset.objects.filter(metadata_only=False).count(), resp.json()["meta"]["total_count"]) + + def test_the_api_should_return_all_datasets_with_metadata_true(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"}) + user = get_user_model().objects.get(username="admin") + token = get_or_create_token(user) + auth_header = f"Bearer {token}" + + url = f"{list_url}?metadata_only=True" + resp = self.api_client.get(url, authentication=auth_header) + self.assertValidJSONResponse(resp) + self.assertEqual(Dataset.objects.filter(metadata_only=True).count(), resp.json()["meta"]["total_count"]) + + def test_the_api_should_return_all_documents_with_metadata_false(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "documents"}) + + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + self.assertEqual(Document.objects.filter(metadata_only=False).count(), resp.json()["meta"]["total_count"]) + + def test_the_api_should_return_all_documents_with_metadata_true(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "documents"}) + + url = f"{list_url}?metadata_only=True" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(Document.objects.filter(metadata_only=True).count(), resp.json()["meta"]["total_count"]) + + def test_the_api_should_return_all_maps_with_metadata_false(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "maps"}) + + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + self.assertEqual(Map.objects.filter(metadata_only=False).count(), resp.json()["meta"]["total_count"]) + + def test_the_api_should_return_all_maps_with_metadata_true(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "maps"}) + + url = f"{list_url}?metadata_only=True" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(Map.objects.filter(metadata_only=True).count(), resp.json()["meta"]["total_count"]) + + def test_api_will_return_a_valid_json_response(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + resp = self.api_client.get(list_url) + self.assertValidJSONResponse(resp) + + def test_will_return_empty_if_the_thesaurus_does_not_exists(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?thesaurus=invalid-identifier" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 0) + + def test_will_return_keywords_for_the_selected_thesaurus_if_exists(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?thesaurus=inspire-theme" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 36) + + def test_will_return_empty_if_the_alt_label_does_not_exists(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?alt_label=invalid-alt_label" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 0) + + def test_will_return_keywords_for_the_selected_alt_label_if_exists(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?alt_label=ac" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 1) + + def test_will_return_empty_if_the_kaywordId_does_not_exists(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?id=12365478954862" + resp = self.api_client.get(url) + print(self.deserialize(resp)) + self.assertValidJSONResponse(resp) + self.assertEqual(resp.json()["meta"]["total_count"], 0) + + @patch("geonode.api.api.get_language") + def test_will_return_expected_keyword_label_for_existing_lang(self, lang): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + lang.return_value = "de" + url = f"{list_url}?thesaurus=inspire-theme" + resp = self.api_client.get(url) + # the german translations exists, for the other labels, the alt_label will be used + expected_labels = [ + "", + "ac", + "Adressen", + "af", + "am", + "au", + "br", + "bu", + "cp", + "ef", + "el", + "er", + "foo_keyword", + "ge", + "gg", + "gn", + "hb", + "hh", + "hy", + "lc", + "lu", + "mf", + "mr", + "nz", + "of", + "oi", + "pd", + "pf", + "ps", + "rs", + "sd", + "so", + "sr", + "su", + "tn", + "us", + ] + actual_labels = [x["alt_label"] for x in self.deserialize(resp)["objects"]] + self.assertValidJSONResponse(resp) + self.assertListEqual(expected_labels, actual_labels) + + @patch("geonode.api.api.get_language") + def test_will_return_default_keyword_label_for_not_existing_lang(self, lang): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + + lang.return_value = "ke" + url = f"{list_url}?thesaurus=inspire-theme" + resp = self.api_client.get(url) + # no translations exists, the alt_label will be used for all keywords + expected_labels = [ + "", + "ac", + "ad", + "af", + "am", + "au", + "br", + "bu", + "cp", + "ef", + "el", + "er", + "foo_keyword", + "ge", + "gg", + "gn", + "hb", + "hh", + "hy", + "lc", + "lu", + "mf", + "mr", + "nz", + "of", + "oi", + "pd", + "pf", + "ps", + "rs", + "sd", + "so", + "sr", + "su", + "tn", + "us", + ] + actual_labels = [x["alt_label"] for x in self.deserialize(resp)["objects"]] + self.assertValidJSONResponse(resp) + self.assertListEqual(expected_labels, actual_labels) + + def test_the_api_should_return_all_map_categories_with_metadata_false(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) + + url = f"{list_url}?type=map" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + actual = sum([x["count"] for x in resp.json()["objects"]]) + self.assertEqual(9, actual) + + def test_the_api_should_return_all_map_categories_with_metadata_true(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) + + x = Map.objects.get(title="map metadata true") + x.metadata_only = False + x.save() + url = f"{list_url}?type=map" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + # by adding a new layer, the total should increase + actual = sum([x["count"] for x in resp.json()["objects"]]) + self.assertEqual(10, actual) + + def test_the_api_should_return_all_document_categories_with_metadata_false(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) + + url = f"{list_url}?type=document" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + actual = sum([x["count"] for x in resp.json()["objects"]]) + self.assertEqual(0, actual) + + def test_the_api_should_return_all_document_categories_with_metadata_true(self): + list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "categories"}) + + x = Document.objects.get(title="doc metadata true") + x.metadata_only = False + x.save() + url = f"{list_url}?type=document" + resp = self.api_client.get(url) + self.assertValidJSONResponse(resp) + # by adding a new layer, the total should increase + actual = sum([x["count"] for x in resp.json()["objects"]]) + self.assertEqual(0, actual) + + +class ThesauriApiTests(GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls): + super().setUpClass() + + cls.user = get_user_model().objects.create(username="user_00") + cls.admin = get_user_model().objects.get(username="admin") + + cls._create_thesauri() + cls._create_resources() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + # remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + + self.api_client = TestApiClient() + + self.assertEqual(self.admin.username, "admin") + self.assertEqual(self.admin.is_superuser, True) + + @classmethod + def _create_thesauri(cls): + cls.thesauri = {} + cls.thesauri_k = {} + + for tn in range(2): + t = Thesaurus.objects.create(identifier=f"t_{tn}", title=f"Thesaurus {tn}") + cls.thesauri[tn] = t + for tl in ( + "en", + "it", + ): + ThesaurusLabel.objects.create(thesaurus=t, lang=tl, label=f"TLabel {tn} {tl}") + + for tkn in range(10): + tk = ThesaurusKeyword.objects.create(thesaurus=t, alt_label=f"alt_tkn{tkn}_t{tn}") + cls.thesauri_k[f"{tn}_{tkn}"] = tk + for tkl in ( + "en", + "it", + ): + ThesaurusKeywordLabel.objects.create(keyword=tk, lang=tkl, label=f"T{tn}_K{tkn}_{tkl}") + + @classmethod + def _create_resources(self): + public_perm_spec = {"users": {"AnonymousUser": ["view_resourcebase"]}, "groups": []} + + for x in range(20): + d: ResourceBase = ResourceBase.objects.create( + title=f"dataset_{x:02}", + uuid=str(uuid4()), + owner=self.user, + abstract=f"Abstract for dataset {x:02}", + subtype="vector", + is_approved=True, + is_published=True, + ) + + # These are the assigned keywords to the Resources + + # RB00 -> T1K0 + # RB01 -> T0K0 T1K0 + # RB02 -> T1K0 + # RB03 -> T0K0 T1K0 + # RB04 -> T1K0 + # RB05 -> T0K0 T1K0 + # RB06 -> T1K0 + # RB07 -> T0K0 T1K0 + # RB08 -> T1K0 T1K1 + # RB09 -> T0K0 T1K0 T1K1 + # RB10 -> T1K1 + # RB11 -> T0K0 T0K1 T1K1 + # RB12 -> T1K1 + # RB13 -> T0K0 T0K1 + # RB14 -> + # RB15 -> T0K0 T0K1 + # RB16 -> + # RB17 -> T0K0 T0K1 + # RB18 -> + # RB19 -> T0K0 T0K1 + + if x % 2 == 1: + print(f"ADDING KEYWORDS {self.thesauri_k['0_0']} to RB {d}") + d.tkeywords.add(self.thesauri_k["0_0"]) + d.save() + if x % 2 == 1 and x > 10: + print(f"ADDING KEYWORDS {self.thesauri_k['0_1']} to RB {d}") + d.tkeywords.add(self.thesauri_k["0_1"]) + d.save() + if x < 10: + print(f"ADDING KEYWORDS {self.thesauri_k['1_0']} to RB {d}") + d.tkeywords.add(self.thesauri_k["1_0"]) + d.save() + if 7 < x < 13: + d.tkeywords.add(self.thesauri_k["1_1"]) + d.save() + + d.set_permissions(public_perm_spec) + + def test_resources_filtered(self): + # list_url = reverse("base-resources", kwargs={"api_name": "api", "resource_name": "base"}) + list_url = reverse("base-resources-list") + + for tks, exp, exp_and in ( + # single filter + (("0_0",), 10, 10), + (("0_1",), 5, 5), + (("1_0",), 10, 10), + (("1_1",), 5, 5), + # same thesaurus: OR + (("0_0", "0_1"), 10, 5), + (("1_0", "1_1"), 13, 2), + # different thesauri: AND + (("0_0", "1_0"), 5, 5), + (("0_1", "1_0"), 0, 0), + (("0_1", "1_0", "1_1"), 1, 0), + (("0_0", "0_1", "1_0", "1_1"), 6, 0), + ): + logger.debug(f"Testing filters for {tks}") + filter = [("filter{tkeywords}", self.thesauri_k[tk].id) for tk in tks] + url = f"{list_url}?{urlencode(filter)}" + resp = self.api_client.get(url).json() + self.assertEqual(exp, resp["total"], f"Unexpected number of resources for default filter {tks}") + + filter = [("filter{tkeywords}", self.thesauri_k[tk].id) for tk in tks] + url = f"{list_url}?{urlencode(filter + [('force_and', True)])}" + resp = self.api_client.get(url).json() + self.assertEqual(exp_and, resp["total"], f"Unexpected number of resources for FORCE_AND filter {tks}") diff --git a/geonode/api/urls.py b/geonode/api/urls.py new file mode 100644 index 0000000..26303ab --- /dev/null +++ b/geonode/api/urls.py @@ -0,0 +1,53 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from tastypie.api import Api +from dynamic_rest import routers +from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView + +from django.urls import path + +from . import api as resources +from . import resourcebase_api as resourcebase_resources + +api = Api(api_name="api") + +api.register(resources.GroupCategoryResource()) +api.register(resources.GroupResource()) +api.register(resources.GroupProfileResource()) +api.register(resources.OwnersResource()) +api.register(resources.ProfileResource()) +api.register(resources.RegionResource()) +api.register(resources.StyleResource()) +api.register(resources.TagResource()) +api.register(resources.ThesaurusKeywordResource()) +api.register(resources.TopicCategoryResource()) +api.register(resourcebase_resources.DocumentResource()) +api.register(resourcebase_resources.FeaturedResourceBaseResource()) +api.register(resourcebase_resources.LayerResource()) +api.register(resourcebase_resources.MapResource()) +api.register(resourcebase_resources.GeoAppResource()) +api.register(resourcebase_resources.ResourceBaseResource()) + +router = routers.DynamicRouter() + +urlpatterns = [ + path("schema/", SpectacularAPIView.as_view(), name="schema"), + path("schema/swagger-ui/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), + path("schema/redoc/", SpectacularRedocView.as_view(url_name="schema"), name="redoc"), +] diff --git a/geonode/api/views.py b/geonode/api/views.py new file mode 100644 index 0000000..b43eb9c --- /dev/null +++ b/geonode/api/views.py @@ -0,0 +1,164 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json + +from django.utils import timezone +from django.http import HttpResponse +from django.contrib.auth import get_user_model +from django.views.decorators.csrf import csrf_exempt + +from guardian.models import Group + +from oauth2_provider.models import AccessToken +from oauth2_provider.exceptions import OAuthToolkitError, FatalClientError +from allauth.account.utils import user_field, user_email, user_username + +from ..utils import json_response +from ..decorators import superuser_or_apiauth +from ..base.auth import get_token_object_from_session, get_auth_token + + +def verify_access_token(request, key): + try: + token = None + if request: + token = get_token_object_from_session(request.session) + if not token or token.key != key: + token = AccessToken.objects.get(token=key) + if not token.is_valid(): + raise OAuthToolkitError("AccessToken is not valid.") + if token.is_expired(): + raise OAuthToolkitError("AccessToken has expired.") + except AccessToken.DoesNotExist: + raise FatalClientError("AccessToken not found at all.") + except Exception: + return None + return token + + +@csrf_exempt +def user_info(request): + user = request.user + + if not user or user.is_anonymous: + out = {"success": False, "status": "error", "errors": {"user": ["User is not authenticated"]}} + return json_response(out, status=401) + + access_token = get_auth_token(user) + + groups = [group.name for group in user.groups.all()] + if user.is_superuser: + groups.append("admin") + + user_info = json.dumps( + { + "sub": str(user.id), + "name": " ".join([user_field(user, "first_name"), user_field(user, "last_name")]), + "given_name": user_field(user, "first_name"), + "family_name": user_field(user, "last_name"), + "email": user_email(user), + "preferred_username": user_username(user), + "groups": groups, + "access_token": str(access_token), + } + ) + + response = HttpResponse(user_info, content_type="application/json") + response["Cache-Control"] = "no-store" + response["Pragma"] = "no-cache" + + return response + + +@csrf_exempt +def verify_token(request): + if request.POST and "token" in request.POST: + token = None + try: + access_token = request.POST.get("token") + token = verify_access_token(request, access_token) + except Exception as e: + return HttpResponse(json.dumps({"error": str(e)}), status=403, content_type="application/json") + + if token: + token_info = json.dumps( + { + "client_id": token.application.client_id, + "user_id": token.user.id, + "username": token.user.username, + "issued_to": token.user.username, + "access_token": access_token, + "email": token.user.email, + "verified_email": "true", + "access_type": "online", + "expires_in": (token.expires - timezone.now()).total_seconds() * 1000, + } + ) + + response = HttpResponse(token_info, content_type="application/json") + response["Authorization"] = f"Bearer {access_token}" + return response + else: + return HttpResponse( + json.dumps({"error": "No access_token from server."}), status=403, content_type="application/json" + ) + + return HttpResponse(json.dumps({"error": "invalid_request"}), status=403, content_type="application/json") + + +@csrf_exempt +@superuser_or_apiauth() +def roles(request): + groups = [group.name for group in Group.objects.all()] + groups.append("admin") + + return HttpResponse(json.dumps({"groups": groups}), content_type="application/json") + + +@csrf_exempt +@superuser_or_apiauth() +def users(request): + user_name = request.path_info.rsplit("/", 1)[-1] + User = get_user_model() + + if user_name is None or not user_name or user_name == "users": + users = [user for user in User.objects.all()] + else: + users = [user for user in User.objects.filter(username=user_name)] + + if not users: + # Try using the user email + users = [user for user in User.objects.filter(email=user_name)] + + json_object = [] + for user in users: + groups = [group.name for group in user.groups.all()] + if user.is_superuser: + groups.append("admin") + + json_object.append({"username": user.username, "groups": groups}) + + return HttpResponse(json.dumps({"users": json_object}), content_type="application/json") + + +@csrf_exempt +@superuser_or_apiauth() +def admin_role(request): + return HttpResponse(json.dumps({"adminRole": "admin"}), content_type="application/json") diff --git a/geonode/apps.py b/geonode/apps.py new file mode 100644 index 0000000..c1c3798 --- /dev/null +++ b/geonode/apps.py @@ -0,0 +1,36 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig as BaseAppConfig + + +def run_setup_hooks(*args, **kwargs): + from django.conf import settings + from .celery_app import app as celery_app + + if celery_app not in settings.INSTALLED_APPS: + settings.INSTALLED_APPS += (celery_app,) + + +class AppConfig(BaseAppConfig): + name = "geonode" + label = "geonode" + + def ready(self): + super().ready() + run_setup_hooks() diff --git a/geonode/assets/__init__.py b/geonode/assets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/assets/admin.py b/geonode/assets/admin.py new file mode 100644 index 0000000..f9f1e9f --- /dev/null +++ b/geonode/assets/admin.py @@ -0,0 +1,55 @@ +import json +import logging +from django.db import models +from django.forms import widgets +from django.contrib import admin + +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import LocalAsset +from geonode.base.models import Link + +logger = logging.getLogger(__name__) + + +class PrettyJSONWidget(widgets.Textarea): + + def format_value(self, value): + try: + value = json.dumps(json.loads(value), indent=2, sort_keys=True) + # these lines will try to adjust size of TextArea to fit to content + row_lengths = [len(r) for r in value.split("\n")] + self.attrs["rows"] = min(max(len(row_lengths) + 2, 10), 30) + self.attrs["cols"] = min(max(max(row_lengths) + 2, 40), 120) + return value + except Exception as e: + logger.warning("Error while formatting JSON: {}".format(e)) + return super(PrettyJSONWidget, self).format_value(value) + + +@admin.register(LocalAsset) +class LocalAssetAdmin(admin.ModelAdmin): + model = LocalAsset + + list_display = ("id", "title", "type", "owner", "created_formatted", "managed", "links", "link0") + list_display_links = ("id", "title") + + formfield_overrides = {models.JSONField: {"widget": PrettyJSONWidget}} + + def created_formatted(self, obj): + return obj.created.strftime("%Y-%m-%d %H:%M:%S") + + def links(self, obj): + return Link.objects.filter(asset=obj).count() + + def link0(self, obj): + link = Link.objects.filter(asset=obj).first() + return f"{link.link_type} {link.extension}: {link.name}" if link else None + + def managed(self, obj) -> bool: + try: + return LocalAssetHandler._is_file_managed(obj.location[0]) + except Exception as e: + logger.error(f"Bad location for asset obj: {e}") + return None + + managed.boolean = True diff --git a/geonode/assets/apps.py b/geonode/assets/apps.py new file mode 100644 index 0000000..c994f7b --- /dev/null +++ b/geonode/assets/apps.py @@ -0,0 +1,35 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + +from geonode.notifications_helper import NotificationsAppConfigBase + + +class BaseAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.assets" + + def ready(self): + super().ready() + run_setup_hooks() + + +def run_setup_hooks(*args, **kwargs): + from geonode.assets.handlers import asset_handler_registry + + asset_handler_registry.init_registry() diff --git a/geonode/assets/handlers.py b/geonode/assets/handlers.py new file mode 100644 index 0000000..2e728ea --- /dev/null +++ b/geonode/assets/handlers.py @@ -0,0 +1,92 @@ +import logging + +from django.conf import settings +from django.http import HttpResponse +from django.utils.module_loading import import_string + +from geonode.assets.models import Asset + +logger = logging.getLogger(__name__) + + +class AssetDownloadHandlerInterface: + + def create_response(self, asset: Asset, attachment: bool = False, basename=None, path=None) -> HttpResponse: + raise NotImplementedError() + + +class AssetHandlerInterface: + + def handled_asset_class(self): + raise NotImplementedError() + + def create(self, title, description, type, owner, *args, **kwargs): + raise NotImplementedError() + + def remove_data(self, asset: Asset, **kwargs): + raise NotImplementedError() + + def replace_data(self, asset: Asset, files: list): + raise NotImplementedError() + + def clone(self, asset: Asset) -> Asset: + """ + Creates a copy in the DB and copies the underlying data as well + """ + raise NotImplementedError() + + def create_link_url(self, asset: Asset) -> str: + raise NotImplementedError() + + def get_download_handler(self, asset: Asset = None) -> AssetDownloadHandlerInterface: + raise NotImplementedError() + + def get_storage_manager(self, asset=None): + raise NotImplementedError() + + +class AssetHandlerRegistry: + _registry = {} + _default_handler = None + + def init_registry(self): + self.register_asset_handlers() + self.set_default_handler() + + def register_asset_handlers(self): + for module_path in settings.ASSET_HANDLERS: + handler = import_string(module_path) + self.register(handler) + logger.info(f"Registered Asset handlers: {', '.join(settings.ASSET_HANDLERS)}") + + def set_default_handler(self): + # check if declared class is registered + for handler in self._registry.values(): + if ".".join([handler.__class__.__module__, handler.__class__.__name__]) == settings.DEFAULT_ASSET_HANDLER: + self._default_handler = handler + break + + if self._default_handler is None: + logger.error(f"Could not set default asset handler class {settings.DEFAULT_ASSET_HANDLER}") + else: + logger.info(f"Default Asset handler {settings.DEFAULT_ASSET_HANDLER}") + + def register(self, asset_handler_class): + self._registry[asset_handler_class.handled_asset_class()] = asset_handler_class() + + def get_default_handler(self) -> AssetHandlerInterface: + return self._default_handler + + def get_handler(self, asset): + asset = asset.get_real_instance() if isinstance(asset, Asset) else asset + asset_cls = asset if isinstance(asset, type) else asset.__class__ + ret = self._registry.get(asset_cls, None) + if not ret: + logger.warning(f"Could not find asset handler for {asset_cls}::{asset.__class__}") + logger.warning("Available asset types:") + for k, v in self._registry.items(): + logger.warning(f"{k} --> {v.__class__.__name__}") + return ret + + +asset_handler_registry = AssetHandlerRegistry() diff --git a/geonode/assets/local.py b/geonode/assets/local.py new file mode 100644 index 0000000..42195db --- /dev/null +++ b/geonode/assets/local.py @@ -0,0 +1,285 @@ +import datetime +import logging +import os +import shutil + +from django.conf import settings +from django.http import HttpResponse, StreamingHttpResponse +from django.urls import reverse +from django_downloadview import DownloadResponse +from zipstream import ZipStream + +from geonode.assets.handlers import asset_handler_registry, AssetHandlerInterface, AssetDownloadHandlerInterface +from geonode.assets.models import LocalAsset +from geonode.storage.manager import DefaultStorageManager, StorageManager +from geonode.utils import build_absolute_uri, mkdtemp + +logger = logging.getLogger(__name__) + +_asset_storage_manager = StorageManager( + concrete_storage_manager=DefaultStorageManager(location=os.path.dirname(settings.ASSETS_ROOT)) +) + + +class DefaultLocalLinkUrlHandler: + def get_link_url(self, asset: LocalAsset): + return build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + + +class IndexLocalLinkUrlHandler: + def get_link_url(self, asset: LocalAsset): + asset = asset.get_real_instance() + if not isinstance(asset, LocalAsset): + raise TypeError("Only localasset are allowed") + return build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + f"/{os.path.basename(asset.location[0])}" + + +class LocalAssetHandler(AssetHandlerInterface): + + link_url_handlers = {"3dtiles": IndexLocalLinkUrlHandler()} + + @staticmethod + def handled_asset_class(): + return LocalAsset + + def get_download_handler(self, asset=None): + return LocalAssetDownloadHandler() + + def get_storage_manager(self, asset): + return _asset_storage_manager + + def get_link_url_handler(self, asset): + return self.link_url_handlers.get(asset.type, None) or DefaultLocalLinkUrlHandler() + + def _create_asset_dir(self): + return os.path.normpath( + mkdtemp(dir=settings.ASSETS_ROOT, prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + ) + + def create(self, title, description, type, owner, files=None, clone_files=False, *args, **kwargs): + if not files: + raise ValueError("File(s) expected") + + if clone_files: + files = self._copy_data(files) + + asset = LocalAsset( + title=title, + description=description, + type=type, + owner=owner, + created=datetime.datetime.now(), + location=files, + ) + asset.save() + return asset + + def remove_data(self, asset: LocalAsset): + """ + Removes the files related to an Asset. + Only files within the Assets directory are removed + """ + asset = self.__force_real_instance(asset) + if self._are_files_managed(asset): + logger.info(f"Removing files for asset {asset.pk}") + base = self._get_managed_dir(asset) + logger.info(f"Removing asset path {base} for asset {asset.pk}") + shutil.rmtree(base) + else: + logger.info(f"Not removing unmanaged files for asset {asset.pk}") + + def replace_data(self, asset: LocalAsset, files: list): + asset = self.__force_real_instance(asset) + self.remove_data(asset) + asset.location = files + asset.save() + + def _copy_data(self, files): + new_path = self._create_asset_dir() + logger.info(f"Copying asset data from {files} into {new_path}") + new_files = [] + for file in files: + if os.path.isdir(file): + dst = os.path.join(new_path, os.path.basename(file)) + logging.info(f"Copying into {dst} directory {file}") + new_dir = shutil.copytree(file, dst) + new_files.append(new_dir) + elif os.path.isfile(file): + logging.info(f"Copying into {new_path} file {os.path.basename(file)}") + new_file = shutil.copy2(file, new_path) + new_files.append(new_file) + else: + logger.warning(f"Not copying path {file}") + + return new_files + + def _clone_data(self, source_dir): + new_path = self._create_asset_dir() + logger.info(f"Cloning asset data from {source_dir} into {new_path}") + + if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None: + # value is always set by default as None + # https://docs.djangoproject.com/en/3.2/ref/settings/#file-upload-directory-permissions + os.chmod(new_path, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) + + shutil.copytree(source_dir, new_path, dirs_exist_ok=True) + + return new_path + + def clone(self, source: LocalAsset) -> LocalAsset: + # get a new asset instance to be edited and stored back + source = self.__force_real_instance(source) + asset = LocalAsset.objects.get(pk=source.pk) + + # only copy files if they are managed + if self._are_files_managed(asset): + base = self._get_managed_dir(asset) + cloned = self._clone_data(base) + asset.location = [os.path.normpath(file).replace(base, cloned) for file in asset.location] + + # it's a polymorphic object, we need to null both IDs + # https://django-polymorphic.readthedocs.io/en/stable/advanced.html#copying-polymorphic-objects + asset.pk = None + asset.id = None + asset.save() + asset.refresh_from_db() + return asset + + def create_download_url(self, asset) -> str: + return build_absolute_uri(reverse("assets-download", args=(asset.pk,))) + + def create_link_url(self, asset) -> str: + return self.get_link_url_handler(asset).get_link_url(asset) + + @classmethod + def _is_file_managed(cls, file) -> bool: + assets_root = os.path.join(os.path.normpath(settings.ASSETS_ROOT), "") + return file.startswith(assets_root) + + @classmethod + def _are_files_managed(cls, asset: LocalAsset) -> bool: + """ + :param asset: asset pointing to the files to be checked + :return: True if all files are managed, False is no file is managed + :raise: ValueError if both managed and unmanaged files are in the list + """ + managed = unmanaged = None + for file in asset.location: + if cls._is_file_managed(file): + managed = True + else: + unmanaged = True + if managed and unmanaged: + logger.error(f"Both managed and unmanaged files are present on Asset {asset.pk}: {asset.location}") + raise ValueError("Both managed and unmanaged files are present") + + return bool(managed) + + @classmethod + def _get_managed_dir(cls, asset): + if not asset.location: + raise ValueError("Asset does not have any associated file") + + assets_root = os.path.normpath(settings.ASSETS_ROOT) + base_common = None + + for file in asset.location: + if not cls._is_file_managed(file): + raise ValueError("Asset is unmanaged") + + norm_file = os.path.normpath(file) + relative = norm_file.removeprefix(assets_root) + base = os.path.split(relative)[0].lstrip("/") + + if base_common: + if base_common != base: + raise ValueError(f"Mismatching base dir in asset files - Asset {asset.pk}") + else: + base_common = base + + managed_dir = os.path.join(assets_root, base_common) + + if not os.path.exists(managed_dir): + raise ValueError(f"Common dir '{managed_dir}' does not exist - Asset {asset.pk}") + + if not os.path.isdir(managed_dir): + raise ValueError(f"Common dir '{managed_dir}' does not seem to be a directory - Asset {asset.pk}") + + if assets_root == managed_dir: # dunno if this can ever happen, but better safe than sorry + raise ValueError(f"Common dir '{managed_dir}' matches the whole Assets dir - Asset {asset.pk}") + + return managed_dir + + @classmethod + def __force_real_instance(cls, asset): + asset = asset.get_real_instance() + if not isinstance(asset, LocalAsset): + raise TypeError(f"Real instance of asset {asset} is not {cls.handled_asset_class()}") + return asset + + +class LocalAssetDownloadHandler(AssetDownloadHandlerInterface): + + def create_response( + self, asset: LocalAsset, attachment: bool = False, basename: str = None, path: str = None + ) -> HttpResponse: + asset = asset.get_real_instance() + if not isinstance(asset, LocalAsset): + raise TypeError("Only localasset are allowed") + if not asset.location: + return HttpResponse("Asset does not contain any data", status=500) + + if len(asset.location) > 1: + logger.warning("TODO: Asset contains more than one file. Download needs to be implemented") + + file0 = asset.location[0] + if not path: # use the file definition + if not os.path.isfile(file0): + logger.warning(f"Default file {file0} not found for asset {asset.id}") + return HttpResponse(f"Default file not found for asset {asset.id}", status=400) + localfile = file0 + + else: # a specific file is requested + if "/../" in path: # we may want to improve fraudolent request detection + logger.warning(f"Tentative path traversal for asset {asset.id}") + return HttpResponse(f"File not found for asset {asset.id}", status=400) + + if os.path.isfile(file0): + dir0 = os.path.dirname(file0) + elif os.path.isdir(file0): + dir0 = file0 + else: + return HttpResponse(f"Unexpected internal location '{file0}' for asset {asset.id}", status=500) + + localfile = os.path.join(dir0, path) + logger.debug(f"Requested path {dir0} + {path}") + + if os.path.isfile(localfile): + filename = os.path.basename(localfile) + orig_base, ext = os.path.splitext(filename) + outname = f"{basename or orig_base or 'file'}{ext}" + match attachment: + case True: + logger.info(f"Zipping file '{localfile}' with name '{orig_base}'") + zs = ZipStream(sized=True).from_path(LocalAssetHandler._get_managed_dir(asset), arcname="/") + # closing zip for all contents to be written + return StreamingHttpResponse( + zs, + content_type="application/zip", + headers={ + "Content-Disposition": f"attachment; filename={orig_base}.zip", + "Content-Length": len(zs), + "Last-Modified": zs.last_modified, + }, + ) + case False: + logger.info(f"Returning file '{localfile}' with name '{outname}'") + return DownloadResponse( + _asset_storage_manager.open(localfile).file, basename=f"{outname}", attachment=False + ) + else: + logger.warning(f"Internal file {localfile} not found for asset {asset.id}") + return HttpResponse(f"Internal file not found for asset {asset.id}", status=404 if path else 500) + + +asset_handler_registry.register(LocalAssetHandler) diff --git a/geonode/assets/management/__init__.py b/geonode/assets/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/assets/management/commands/__init__.py b/geonode/assets/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/assets/management/commands/migrate_file_to_assets.py b/geonode/assets/management/commands/migrate_file_to_assets.py new file mode 100644 index 0000000..de8e905 --- /dev/null +++ b/geonode/assets/management/commands/migrate_file_to_assets.py @@ -0,0 +1,221 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import sys +import shutil +import logging +from django.conf import settings +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import LocalAsset +from geonode.base.management.commands.helpers import confirm +from django.core.management.base import BaseCommand +from geonode.geoserver.helpers import gs_catalog + +from geonode.base.models import ResourceBase + +logger = logging.getLogger() +handler = logging.StreamHandler(sys.stdout) +logger.setLevel(logging.INFO) +formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") +handler.setFormatter(formatter) +logger.addHandler(handler) + + +class Command(BaseCommand): + + help = """ + Migrate the files from MEDIA_ROOT to ASSETS_ROOT, update LocalAsset.location and GeoServer + REF: https://github.com/GeoNode/geonode/issues/12326 + """ + + def add_arguments(self, parser): + parser.add_argument( + '-n', + '--no-input', + dest='noinput', + default=False, + action='store_true', + help='Does not ask for confirmation for the run' + ) + parser.add_argument( + '-d', + '--dryrun', + dest='dryrun', + default=False, + action='store_true', + help='Perform a dryrun, will show the File to be moved, their new path and the file to be deleted' + ) + + def handle(self, **options): + + # copied from LocalAssetHandler and fixed + def is_file_managed(loc) -> bool: + return loc.startswith(os.path.join(os.path.normpath(settings.ASSETS_ROOT), "")) + + # copied from LocalAssetHandler + def are_files_managed(asset: LocalAsset) -> bool: + """ + :param asset: asset pointing to the files to be checked + :return: True if all files are managed, False is no file is managed + :raise: ValueError if both managed and unmanaged files are in the list + """ + managed = unmanaged = None + for file in asset.location: + if is_file_managed(file): + managed = True + else: + unmanaged = True + if managed and unmanaged: + logger.error(f"Both managed and unmanaged files are present on Asset {asset.pk}: {asset.location}") + raise ValueError("Both managed and unmanaged files are present") + + return bool(managed) + + dorun = not options.get('dryrun') + + if dorun and not options.get('noinput'): + question = "By running this command you are going to move all the files of your Resources into the ASSETS_ROOT. Do you want to continue?" + if not confirm(question, resp=True): + return + + logger.info("Retrieving LocalAssets...") + tot = LocalAsset.objects.count() + cnt = 0 + + for asset in LocalAsset.objects.iterator(): + cnt += 1 + + logger.info(f'#### {cnt}/{tot} - Processing asset {asset.id}: "{asset.title}"') + for loc in asset.location: + logger.info(f" - File: {loc}") + + try: + if are_files_managed(asset): + logger.info("Asset is already managed, skipping...") + continue + except ValueError as e: + logger.error("Some files are in the assets dir, other are not. You may need to fix this by hand.") + logger.exception(e) + continue + + source = os.path.dirname(asset.location[0]) + if not os.path.exists(source): + logger.error("Source path of the file for Asset does not exists, skipping...") + continue + + try: + handler = LocalAssetHandler() + dest_dir = handler._create_asset_dir() + dest = None + + logger.info(f"Moving data to the asset folder {dest_dir}") + if len(asset.location) == 1: + # In older installations, all documents are stored in a single folder, e.g. + # oldpath = {MEDIA_ROOT}/documents/document/file.extension + # Instead of moving the entire folder, we can simply move the individual document. + # This approach prevents the risk of breaking the other documents + # that are stored in the same folder + + logger.info(f"Moving dir: {asset.location[0]} -> {dest_dir}") + if dorun: + dest = shutil.move(asset.location[0], dest_dir) + else: + # values only used in dryrun mode for logging + dest = f"[DRYRUN] {os.path.join(dest_dir, os.path.basename(asset.location[0]))}" + asset.location = [dest] + else: + logger.info(f"Moving file {source} -> {dest_dir}") + if dorun: + dest = shutil.move(source, dest_dir) + else: + # values only used in dryrun mode for logging + dest = f"[DRYRUN] {os.path.join(dest_dir, os.path.basename(source))}" + asset.location = [x.replace(source, dest) for x in asset.location] + + logger.info(f"New location path: {dest}") + logger.info("New asset.location:") + for loc in asset.location: + logger.info(f" - {loc}") + + if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None: + logger.info(f"Setting folder perms to {settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS}") + if dorun: + os.chmod(os.path.dirname(dest), settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) + + if settings.FILE_UPLOAD_PERMISSIONS is not None: + logger.info(f"Setting file perms to {settings.FILE_UPLOAD_PERMISSIONS}") + if dorun: + os.chmod(dest, settings.FILE_UPLOAD_PERMISSIONS) + + except Exception as e: + logger.error(e) + continue + + if dorun: + logger.info("Updating asset.location in database") + asset.save() + + logger.info("Checking if geoserver should be updated") + asset_obj = asset.link_set.values_list('resource', flat=True).first() + if not asset_obj: + logger.warning("No resources connected to the asset, skipping resource update...") + continue + resource = ResourceBase.objects.get(pk=asset_obj) + logger.info(f'Related resource id: {resource.id} type: {resource.resource_type}/{resource.subtype} "{resource.title}"') + + if resource.subtype == 'raster': + logger.info(f"Checking GeoServer layer {resource.get_real_instance().alternate}") + + # look for store + gs_layer = gs_catalog.get_layer(resource.get_real_instance().alternate) + if not gs_layer: + logger.info(f"Layer not found in GeoServer: {resource.get_real_instance().alternate}") + continue + if not gs_layer.resource: + logger.info(f"Resource not attached to layer in GeoServer: {resource.get_real_instance().alternate}") + continue + if not gs_layer.resource.store: + logger.info(f"Store not attached to resource in GeoServer: {resource.get_real_instance().alternate}") + continue + store_to_update = gs_layer.resource.store + + # look for the file to set in geoserver + new_url = None + for loc in asset.location: + if not new_url and os.path.splitext(loc)[1] in [".tiff", ".tif", ".geotiff", ".geotif"]: + logger.info(f" -> Using file : {loc}") + new_url = loc + else: + logger.info(f" - Skipping file: {loc}") + + # update store in geoserver + if new_url: + store_to_update.url = f"file:{new_url}" + try: + logger.info(f"Updating GeoServer store {store_to_update.name}") + if dorun: + gs_catalog.save(store_to_update) + except Exception: + logger.error(f"Error during GeoServer update for resource {resource}, please check GeoServer logs") + logger.info("Geoserver Updated") + else: + logger.warning(f"No valid file found to set in GeoServer in {asset.location}") + + logger.info(f"Migration completed{'' if dorun else ' -- DRYRUN'}") diff --git a/geonode/assets/migrations/0001_initial.py b/geonode/assets/migrations/0001_initial.py new file mode 100644 index 0000000..8a1ef08 --- /dev/null +++ b/geonode/assets/migrations/0001_initial.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2.9 on 2024-04-24 10:02 + +from django.conf import settings +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("base", "0091_alter_hierarchicalkeyword_slug"), + ] + + operations = [ + migrations.CreateModel( + name="Asset", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("title", models.CharField(max_length=255)), + ("description", models.TextField(blank=True, null=True)), + ("type", models.CharField(max_length=255)), + ("created", models.DateTimeField(auto_now_add=True)), + ("owner", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "polymorphic_ctype", + models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="polymorphic_%(app_label)s.%(class)s_set+", + to="contenttypes.contenttype", + ), + ), + ], + options={ + "verbose_name_plural": "Assets", + }, + ), + migrations.CreateModel( + name="LocalAsset", + fields=[ + ( + "asset_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="assets.asset", + ), + ), + ("location", models.JSONField(blank=True, default=list)), + ], + options={ + "verbose_name_plural": "Local assets", + }, + bases=("assets.asset",), + ), + ] diff --git a/geonode/assets/migrations/__init__.py b/geonode/assets/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/assets/models.py b/geonode/assets/models.py new file mode 100644 index 0000000..2d81791 --- /dev/null +++ b/geonode/assets/models.py @@ -0,0 +1,60 @@ +from django.db import models +from polymorphic.managers import PolymorphicManager +from polymorphic.models import PolymorphicModel +from django.db.models import signals +from django.contrib.auth import get_user_model + + +class AssetPolymorphicManager(PolymorphicManager): + """ + This override is required for the dump procedure. + Otherwise django is not able to dump the base objects + and will be upcasted to polymorphic models + https://github.com/jazzband/django-polymorphic/blob/cfd49b26d580d99b00dcd43a02409ce439a2c78f/polymorphic/base.py#L161-L175 + """ + + def get_queryset(self): + return super().get_queryset().non_polymorphic() + + +class Asset(PolymorphicModel): + """ + A generic data linked to a ResourceBase + """ + + title = models.CharField(max_length=255, null=False, blank=False) + description = models.TextField(null=True, blank=True) + type = models.CharField(max_length=255, null=False, blank=False) + owner = models.ForeignKey(get_user_model(), null=False, blank=False, on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + + objects = AssetPolymorphicManager() + + class Meta: + verbose_name_plural = "Assets" + + def __str__(self) -> str: + return super().__str__() + + +class LocalAsset(Asset): + """ + Local resource, will replace the files + """ + + location = models.JSONField(default=list, blank=True) + + class Meta: + verbose_name_plural = "Local assets" + + def __str__(self) -> str: + return f"{self.__class__.__name__}: {self.type}|{self.title}" + + +def cleanup_asset_data(instance, *args, **kwargs): + from geonode.assets.handlers import asset_handler_registry + + asset_handler_registry.get_handler(instance).remove_data(instance) + + +signals.post_delete.connect(cleanup_asset_data, sender=LocalAsset) diff --git a/geonode/assets/serializers.py b/geonode/assets/serializers.py new file mode 100644 index 0000000..999cf80 --- /dev/null +++ b/geonode/assets/serializers.py @@ -0,0 +1,78 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.contrib.auth import get_user_model + +from dynamic_rest.serializers import DynamicModelSerializer +from dynamic_rest.fields.fields import DynamicComputedField + +from geonode.assets.models import ( + Asset, + LocalAsset, +) + +logger = logging.getLogger(__name__) + + +class ClassTypeField(DynamicComputedField): + + def get_attribute(self, instance): + return type(instance).__name__ + + +class SimpleUserSerializer(DynamicModelSerializer): + class Meta: + model = get_user_model() + name = "user" + fields = ("pk", "username") + + +class AssetSubclassField(DynamicComputedField): + """ + Just an ugly hack. + TODO: We need a way to automatically use a proper serializer for each Asset subclass + in order to render different instances in a list + """ + + def get_attribute(self, instance): + if type(instance).__name__ == "LocalAsset": + return {"locations": instance.location} + + return None + + +class AssetSerializer(DynamicModelSerializer): + + owner = SimpleUserSerializer(embed=False) + asset_type = ClassTypeField() + subinfo = AssetSubclassField() + + class Meta: + model = Asset + name = "asset" + # fields = ("pk", "title", "description", "type", "owner", "created") + fields = ("pk", "title", "description", "type", "owner", "created", "asset_type", "subinfo") + + +class LocalAssetSerializer(AssetSerializer): + class Meta(AssetSerializer.Meta): + model = LocalAsset + name = "local_asset" + fields = AssetSerializer.Meta.fields + ("location",) diff --git a/geonode/assets/tests.py b/geonode/assets/tests.py new file mode 100644 index 0000000..d281312 --- /dev/null +++ b/geonode/assets/tests.py @@ -0,0 +1,316 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import logging +import shutil +import io +import json + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.http import StreamingHttpResponse +from django.urls import reverse + +from rest_framework.test import APITestCase + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import Asset, LocalAsset + +logger = logging.getLogger(__name__) + +ONE_JSON = os.path.join(os.path.dirname(__file__), "tests/data/one.json") +TWO_JSON = os.path.join(os.path.dirname(__file__), "tests/data/two.json") +THREE_JSON = os.path.join(os.path.dirname(__file__), "tests/data/three.json") + + +class AssetsTests(APITestCase): + + def test_handler_registry(self): + # Test registry + self.assertIsNotNone(asset_handler_registry) + # Test default handler + asset_handler = asset_handler_registry.get_default_handler() + self.assertIsNotNone(asset_handler) + self.assertIsInstance(asset_handler, LocalAssetHandler, "Bad default Asset handler found") + # Test None + self.assertIsNone(asset_handler_registry.get_handler(None)) + # Test class without handler + self.assertIsNone(asset_handler_registry.get_handler(AssetsTests)) + + def test_creation_and_delete_data_cloned(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + assets_root = os.path.normpath(settings.ASSETS_ROOT) + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = LocalAsset.objects.get(pk=asset.pk) + self.assertIsNotNone(reloaded) + self.assertIsInstance(reloaded, LocalAsset) + file = reloaded.location[0] + self.assertTrue(os.path.exists(file), "Asset file does not exist") + self.assertTrue( + os.path.normpath(file).startswith(os.path.normpath(assets_root)), + f"Asset file is not inside the assets root: {file}", + ) + + cloned_file = file + reloaded.delete() + self.assertFalse(Asset.objects.filter(pk=asset.pk).exists()) + self.assertFalse(os.path.exists(cloned_file)) + self.assertFalse(os.path.exists(os.path.dirname(cloned_file))) + self.assertTrue(os.path.exists(ONE_JSON)) + + def test_creation_and_delete_data_external(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=False, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = LocalAsset.objects.get(pk=asset.pk) + self.assertIsNotNone(reloaded) + self.assertIsInstance(reloaded, LocalAsset) + file = reloaded.location[0] + self.assertEqual(ONE_JSON, file) + + reloaded.delete() + self.assertFalse(Asset.objects.filter(pk=asset.pk).exists()) + self.assertTrue(os.path.exists(ONE_JSON)) + + def test_clone_and_delete_data_managed(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = LocalAsset.objects.get(pk=asset.pk) + cloned = asset_handler.clone(reloaded) + self.assertNotEqual(reloaded.pk, cloned.pk) + + reloaded_file = os.path.normpath(reloaded.location[0]) + cloned_file = os.path.normpath(cloned.location[0]) + + self.assertNotEqual(reloaded_file, cloned_file) + self.assertTrue(os.path.exists(reloaded_file)) + self.assertTrue(os.path.exists(cloned_file)) + + reloaded.delete() + self.assertFalse(os.path.exists(reloaded_file)) + self.assertTrue(os.path.exists(cloned_file)) + + cloned.delete() + self.assertFalse(os.path.exists(cloned_file)) + + def test_clone_and_delete_data_unmanaged(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=False, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = LocalAsset.objects.get(pk=asset.pk) + cloned = asset_handler.clone(reloaded) + + self.assertEqual(reloaded.location[0], cloned.location[0]) + self.assertTrue(os.path.exists(reloaded.location[0])) + + reloaded.delete() + self.assertTrue(os.path.exists(reloaded.location[0])) + + cloned.delete() + self.assertTrue(os.path.exists(reloaded.location[0])) + + def test_clone_mixed_data(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + managed_asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[ONE_JSON], + clone_files=True, + ) + managed_asset.save() + + # TODO: dunno if mixed files should be allowed at all + mixed_asset = asset_handler.create( + title="Mixed Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[THREE_JSON, managed_asset.location[0]], + clone_files=False, # let's keep both managed and unmanaged together + ) + + reloaded = Asset.objects.get(pk=mixed_asset.pk) + + try: + asset_handler.clone(reloaded) + self.fail("A mixed LocalAsset has been cloned") + except ValueError: + pass + + managed_asset.delete() + + try: + mixed_asset.delete() + self.fail("Missed mixed LocalAsset detection") + except ValueError: + pass + + +class AssetsDownloadTests(APITestCase): + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + def _get_streaming_content(self, response): + with io.BytesIO(b"".join(response.streaming_content)) as buf_bytes: + return buf_bytes.read() + + def test_download_file(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + self.assertTrue(self.client.login(username="admin", password="admin"), "Login failed") + + asset = self._setup_test(u) + + for path, key in ((None, "one"), ("one.json", "one"), ("two.json", "two"), ("subdir/three.json", "three")): + # args = [asset.pk, path] if path else [asset.pk] + args = {"pk": asset.pk, "path": path} if path else {"pk": asset.pk} + logger.info(f"*** Testing path '{path}' args {args}") + url = reverse("assets-link", kwargs=args) + logger.info(f"REVERSE url is {url}") + response = self.client.get(url) + content = self._get_streaming_content(response) + rjson = json.loads(content) + self.assertEqual(response.status_code, 200) + self.assertIn(key, rjson, f"Key '{key}' not found in path '{path}': {rjson} URL {url}") + logger.info(f"Test for path '{path}' OK") + + def test_download_with_attachment(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + self.assertTrue(self.client.login(username="admin", password="admin"), "Login failed") + + for key, el in (("one", ONE_JSON), ("two", TWO_JSON), ("three", THREE_JSON)): + asset = self._setup_test(u, _file=el) + + url = reverse("assets-download", kwargs={"pk": asset.pk}) + logger.info(f"REVERSE url is {url}") + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertTrue(isinstance(response, StreamingHttpResponse)) + self.assertEqual(response.get("Content-Disposition"), f"attachment; filename={key}.zip") + + def test_download_with_attachment_with_no_download_perms(self): + """ + any user without download permissions + cannot download the asset + """ + from geonode.resource.manager import resource_manager + from geonode.layers.models import Dataset + from guardian.shortcuts import get_anonymous_user + + bobby, _ = get_user_model().objects.get_or_create(username="bobby") + + self.client.force_login(get_user_model().objects.get(username="norman")) + for _, el in (("one", ONE_JSON), ("two", TWO_JSON), ("three", THREE_JSON)): + asset = self._setup_test(bobby, _file=el) + resource = None + try: + # creating resoruce + resource = resource_manager.create( + None, resource_type=Dataset, defaults={"owner": bobby, "asset": asset} + ) + # getting perms + # remove perms for user bobby + resource_perms = resource.get_all_level_info() + resource_perms["users"][bobby] = ["view_resourcebase"] + # and anonymous just to be sure + resource_perms["groups"][get_anonymous_user().groups.first()] = ["view_resourcebase"] + # resetting the permissions for the resource + resource_manager.set_permissions(resource.uuid, instance=resource, permissions=resource_perms) + url = reverse("assets-download", kwargs={"pk": asset.pk}) + response = self.client.get(url) + self.assertEqual(response.status_code, 403) + except Exception as e: + logger.exception(e) + finally: + if resource: + resource.delete() + + def _setup_test(self, u, _file=ONE_JSON): + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[_file], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = LocalAsset.objects.get(pk=asset.pk) + + # put two more files in the asset dir + asset_dir = os.path.dirname(reloaded.location[0]) + sub_dir = os.path.join(asset_dir, "subdir") + os.mkdir(sub_dir) + shutil.copy(TWO_JSON, asset_dir) + shutil.copy(THREE_JSON, sub_dir) + return asset diff --git a/geonode/assets/tests/data/one.json b/geonode/assets/tests/data/one.json new file mode 100644 index 0000000..27a1ec0 --- /dev/null +++ b/geonode/assets/tests/data/one.json @@ -0,0 +1,3 @@ +{ + "one": 1 +} \ No newline at end of file diff --git a/geonode/assets/tests/data/three.json b/geonode/assets/tests/data/three.json new file mode 100644 index 0000000..8ced1fb --- /dev/null +++ b/geonode/assets/tests/data/three.json @@ -0,0 +1,3 @@ +{ + "three": 3 +} \ No newline at end of file diff --git a/geonode/assets/tests/data/two.json b/geonode/assets/tests/data/two.json new file mode 100644 index 0000000..c8b2fcf --- /dev/null +++ b/geonode/assets/tests/data/two.json @@ -0,0 +1,3 @@ +{ + "two": 2 +} \ No newline at end of file diff --git a/geonode/assets/urls.py b/geonode/assets/urls.py new file mode 100644 index 0000000..956854d --- /dev/null +++ b/geonode/assets/urls.py @@ -0,0 +1,25 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.api.urls import router + +from geonode.assets import views + +router.register(r"assets", views.AssetViewSet, "assets") + +urlpatterns = [] diff --git a/geonode/assets/utils.py b/geonode/assets/utils.py new file mode 100644 index 0000000..4354ec0 --- /dev/null +++ b/geonode/assets/utils.py @@ -0,0 +1,168 @@ +import logging +import os.path + +from django.http import HttpResponse + +from django.core.exceptions import PermissionDenied +from geonode.security.permissions import DOWNLOAD_PERMISSIONS +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.models import Asset +from geonode.base.models import ResourceBase, Link +from geonode.security.utils import get_visible_resources + + +logger = logging.getLogger(__name__) + + +def get_perms_response(request, asset: Asset): + user = request.user + + # quick check + is_admin = user.is_superuser if user and user.is_authenticated else False + if is_admin or user == asset.owner: + logger.debug("Asset: access allowed by user") + return None + + visibile_res = get_visible_resources(queryset=ResourceBase.objects.filter(link__asset=asset), user=request.user) + + logger.warning("TODO: implement permission check") + if visibile_res.exists(): + if user.has_perm(DOWNLOAD_PERMISSIONS[0]): + logger.debug("Asset: access allowed by Resource") + return None + raise PermissionDenied + elif user and user.is_authenticated: + return HttpResponse(status=403) + else: + return HttpResponse(status=401) + + +def get_default_asset(resource: ResourceBase, link_type=None) -> Asset or None: + """ + Get the default asset for a ResourceBase. + + In this first implementation we select the first one -- + in the future there may be further flags to identify the preferred one + """ + filters = {"link__resource": resource} + if link_type: + filters["link__link_type"] = link_type + asset = Asset.objects.filter(**filters).first() + return asset.get_real_instance() if asset else None + + +DEFAULT_TYPES = {"image": ["jpg", "jpeg", "gif", "png", "bmp", "svg"]} + + +def find_type(ext): + return next((datatype for datatype, extensions in DEFAULT_TYPES.items() if ext.lower() in extensions), None) + + +def create_link(resource, asset, link_type=None, extension=None, name=None, mime=None, asset_handler=None, **kwargs): + asset = asset.get_real_instance() + asset_handler = asset_handler or asset_handler_registry.get_handler(asset) + + if not link_type or not extension or not name: + fallback_name, fallback_ext = ( + os.path.splitext(os.path.basename(asset.location[0])) if len(asset.location) else (None, None) + ) + if fallback_ext: + fallback_ext = fallback_ext.lstrip(".") + link_type = link_type or find_type(fallback_ext) if fallback_ext else None + + link = Link( + resource=resource, + asset=asset, + url=asset_handler.create_link_url(asset), + extension=extension or fallback_ext or "Unknown", + link_type=link_type or "data", + name=name or fallback_name or asset.title, + mime=mime or "", + ) + link.save() + return link + + +def create_asset_and_link( + resource, + owner, + files: list, + handler=None, + title=None, + description=None, + link_type=None, + extension=None, + asset_type=None, + mime=None, + clone_files: bool = True, +) -> tuple[Asset, Link]: + + asset_handler = handler or asset_handler_registry.get_default_handler() + asset = link = None + try: + default_title, default_ext = os.path.splitext(next(f for f in files)) if len(files) else (None, None) + if default_ext: + default_ext = default_ext.lstrip(".") + link_type = link_type or find_type(default_ext) if default_ext else None + + asset = asset_handler.create( + title=title or default_title or "Unknown", + description=description or asset_type or "Unknown", + type=asset_type or "Unknown", + owner=owner, + files=files, + clone_files=clone_files, + ) + + link = create_link( + resource, + asset, + asset_handler=asset_handler, + link_type=link_type, + extension=extension, + name=title, + mime=mime, + ) + + return asset, link + except Exception as e: + logger.error(f"Error creating Asset for resource {resource}: {e}", exc_info=e) + rollback_asset_and_link(asset, link) + raise Exception(f"Error creating asset: {e}") + + +def create_asset_and_link_dict(resource, values: dict, clone_files=True): + return create_asset_and_link( + resource, + values["owner"], + values["files"], + title=values.pop("data_title", None), + description=values.pop("description", None), + link_type=values.pop("link_type", None), + extension=values.pop("extension", None), + asset_type=values.pop("data_type", None), + clone_files=clone_files, + ) + + +def copy_assets_and_links(resource, target=None) -> list: + assets_and_links = [] + links_with_assets = Link.objects.filter(resource=resource, asset__isnull=False).prefetch_related("asset") + + for link in links_with_assets: + link.asset = asset_handler_registry.get_handler(link.asset).clone(link.asset) + link.pk = None + link.resource = target + link.save() + assets_and_links.append((link.asset, link)) + return assets_and_links + + +def rollback_asset_and_link(asset, link): + try: + if link: + link.delete() + if asset: + asset.delete() # TODO: make sure we are only deleting from DB and not also the stored data + except Exception as e: + logger.error(f"Could not rollback asset[{asset}] and link[{link}]", exc_info=e) diff --git a/geonode/assets/views.py b/geonode/assets/views.py new file mode 100644 index 0000000..a12c16d --- /dev/null +++ b/geonode/assets/views.py @@ -0,0 +1,112 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.shortcuts import get_object_or_404 +from dynamic_rest.viewsets import DynamicModelViewSet +from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter + +from oauth2_provider.contrib.rest_framework import OAuth2Authentication + +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.authentication import SessionAuthentication, BasicAuthentication + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.serializers import AssetSerializer +from geonode.assets.utils import get_perms_response +from geonode.assets.models import Asset + +from geonode.base.api.filters import ( + DynamicSearchFilter, +) +from geonode.base.api.pagination import GeoNodeApiPagination + +logger = logging.getLogger(__name__) + + +class AssetViewSet(DynamicModelViewSet): + """ + API endpoint that allows Assets to be viewed or edited. + """ + + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [IsAuthenticatedOrReadOnly] + filter_backends = [ + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + # TODO: add filtering by owner / admin + ] + queryset = Asset.objects.all().order_by("-created") + serializer_class = AssetSerializer # TODO: appropriate Serializer should be switched for each Asset instance + pagination_class = GeoNodeApiPagination + + def list(self, request, *args, **kwargs): + """ + Only for lists, allows access to Assets only to owned ones, or to all of them if the user is an admin + """ + queryset = self.filter_queryset(self.get_queryset()) + + user = request.user + is_admin = user.is_superuser if user and user.is_authenticated else False + + if is_admin: + pass + elif user and user.is_authenticated: + queryset = queryset.filter(owner=user) + else: + queryset = queryset.none() + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + + def _get_file(self, request, pk, attachment: bool = False, path=None): + asset = get_object_or_404(Asset, pk=pk) + if bad_response := get_perms_response(request, asset): + return bad_response + asset_handler = asset_handler_registry.get_handler(asset) + # TODO: register_event(request, EventType.EVENT_DOWNLOAD, asset) + return asset_handler.get_download_handler(asset).create_response(asset, path=path, attachment=attachment) + + @action( + detail=False, + url_path="(?P\d+)/download(?:/(?P.*))?", # noqa + # url_name="asset-download", + methods=["get"], + ) + def download(self, request, pk=None, path=None, *args, **kwargs): + return self._get_file(request, pk, attachment=True, path=path) + + @action( + detail=False, + url_path="(?P\d+)/link(?:/(?P.*))?", # noqa + # url_name="asset-link", + methods=["get"], + ) + def link(self, request, pk=None, path=None, *args, **kwargs): + logger.warning(f"REQUESTED ASSET LINK FOR PK:{pk} PATH:{path}") + return self._get_file(request, pk, attachment=False, path=path) diff --git a/geonode/base/__init__.py b/geonode/base/__init__.py new file mode 100644 index 0000000..dba87a5 --- /dev/null +++ b/geonode/base/__init__.py @@ -0,0 +1,80 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from functools import wraps + +from django.conf import settings + + +def register_url_event(event_type=None): + """ + Decorator on views, which will register url event + + usage: + >> register_url_event()(TemplateView.view_as_view()) + + """ + + def _register_url_event(view): + @wraps(view) + def inner(*args, **kwargs): + if settings.MONITORING_ENABLED: + request = args[0] + register_event(request, event_type or "view", request.path) + return view(*args, **kwargs) + + return inner + + return _register_url_event + + +def register_event(request, event_type, resource): + """ + Wrapper function to be used inside views to collect event and resource + + @param request Request object + @param event_type name of event type + @param resource string (then resource type will be url) or Resource instance + + >>> from geonode.base import register_event + >>> def view(request): + register_event(request, 'view', layer) + """ + if not settings.MONITORING_ENABLED: + return + + from geonode.base.models import ResourceBase + + if isinstance(resource, str): + resource_type = "url" + resource_name = request.path + resource_id = None + elif isinstance(resource, ResourceBase): + resource_type = resource.__class__._meta.verbose_name_raw + resource_name = getattr(resource, "alternate", None) or resource.title + resource_id = resource.id + else: + raise ValueError(f"Invalid resource: {resource}") + if request and hasattr(request, "register_event"): + request.register_event(event_type, resource_type, resource_name, resource_id) + + +def register_proxy_event(request): + """ + Process request to geoserver proxy. Extract layer and ows type + """ diff --git a/geonode/base/admin.py b/geonode/base/admin.py new file mode 100644 index 0000000..0f6c4ee --- /dev/null +++ b/geonode/base/admin.py @@ -0,0 +1,354 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django import forms +from django.contrib import admin +from django.conf import settings +from django.shortcuts import redirect, render +from django.urls import path +from dal import autocomplete +from taggit.forms import TagField +from django.core.management import call_command +from slugify import slugify +from django.contrib import messages + +from treebeard.admin import TreeAdmin +from treebeard.forms import movenodeform_factory + +from modeltranslation.admin import TabbedTranslationAdmin + +from geonode.base.models import ( + TopicCategory, + SpatialRepresentationType, + Region, + RestrictionCodeType, + ContactRole, + Link, + License, + HierarchicalKeyword, + MenuPlaceholder, + Menu, + MenuItem, + Configuration, + Thesaurus, + ThesaurusLabel, + ThesaurusKeyword, + ThesaurusKeywordLabel, +) + +from geonode.base.forms import BatchEditForm, ThesaurusImportForm, UserAndGroupPermissionsForm +from geonode.base.widgets import TaggitSelect2Custom + + +def metadata_batch_edit(modeladmin, request, queryset): + ids = ",".join(str(element.pk) for element in queryset) + resource = queryset[0].class_name.lower() + form = BatchEditForm({"ids": ids}) + name_space_mapper = { + "dataset": "dataset_batch_metadata", + "map": "map_batch_metadata", + "document": "document_batch_metadata", + } + + try: + name_space = name_space_mapper[resource] + except KeyError: + name_space = None + + return render( + request, "base/batch_edit.html", context={"form": form, "ids": ids, "model": resource, "name_space": name_space} + ) + + +metadata_batch_edit.short_description = "Metadata batch edit" + + +def set_user_and_group_dataset_permission(modeladmin, request, queryset): + ids = ",".join(str(element.pk) for element in queryset) + resource = queryset[0].__class__.__name__.lower() + + model_mapper = {"profile": "people", "groupprofile": "groups"} + + form = UserAndGroupPermissionsForm( + { + "permission_type": "view", + "mode": "set", + "ids": ids, + } + ) + + return render( + request, "base/user_and_group_permissions.html", context={"form": form, "model": model_mapper[resource]} + ) + + +set_user_and_group_dataset_permission.short_description = "Set layer permissions" + + +class LicenseAdmin(TabbedTranslationAdmin): + model = License + list_display = ("id", "name") + list_display_links = ("name",) + + +class TopicCategoryAdmin(TabbedTranslationAdmin): + model = TopicCategory + list_display_links = ("identifier",) + list_display = ("identifier", "description", "gn_description", "fa_class", "is_choice") + if settings.MODIFY_TOPICCATEGORY is False: + exclude = ( + "identifier", + "description", + ) + + def has_add_permission(self, request): + # the records are from the standard TC 211 list, so no way to add + if settings.MODIFY_TOPICCATEGORY: + return True + else: + return False + + def has_delete_permission(self, request, obj=None): + # the records are from the standard TC 211 list, so no way to remove + if settings.MODIFY_TOPICCATEGORY: + return True + else: + return False + + +class RegionAdmin(TabbedTranslationAdmin): + model = Region + list_display_links = ("name",) + list_display = ("code", "name", "parent") + search_fields = ( + "code", + "name", + ) + group_fieldsets = True + + +class SpatialRepresentationTypeAdmin(TabbedTranslationAdmin): + model = SpatialRepresentationType + list_display_links = ("identifier",) + list_display = ("identifier", "description", "gn_description", "is_choice") + + def has_add_permission(self, request): + # the records are from the standard TC 211 list, so no way to add + return False + + def has_delete_permission(self, request, obj=None): + # the records are from the standard TC 211 list, so no way to remove + return False + + +class RestrictionCodeTypeAdmin(TabbedTranslationAdmin): + model = RestrictionCodeType + list_display_links = ("identifier",) + list_display = ("identifier", "description", "gn_description", "is_choice") + + def has_add_permission(self, request): + # the records are from the standard TC 211 list, so no way to add + return False + + def has_delete_permission(self, request, obj=None): + # the records are from the standard TC 211 list, so no way to remove + return False + + +class ContactRoleAdmin(admin.ModelAdmin): + model = ContactRole + list_display_links = ("id",) + list_display = ("id", "contact", "resource", "role") + list_editable = ("contact", "resource", "role") + form = forms.modelform_factory(ContactRole, fields="__all__") + + +class LinkAdmin(admin.ModelAdmin): + model = Link + list_display_links = ("id",) + list_display = ("id", "resource", "extension", "link_type", "name", "mime") + list_filter = ("resource", "extension", "link_type", "mime") + search_fields = ( + "name", + "resource__title", + ) + form = forms.modelform_factory(Link, fields="__all__") + + +class HierarchicalKeywordAdmin(TreeAdmin): + search_fields = ("name",) + form = movenodeform_factory(HierarchicalKeyword) + + +class MenuPlaceholderAdmin(admin.ModelAdmin): + model = MenuPlaceholder + list_display = ("name",) + + +class MenuAdmin(admin.ModelAdmin): + model = Menu + list_display = ("title", "placeholder", "order") + + +class MenuItemAdmin(admin.ModelAdmin): + model = MenuItem + list_display = ("title", "menu", "order", "blank_target", "url") + + +class ConfigurationAdmin(admin.ModelAdmin): + model = Configuration + + def has_delete_permission(self, request, obj=None): + # Disable delete action of Singleton model, since "delete selected objects" uses QuerysSet.delete() + # instead of Model.delete() + return False + + def get_form(self, request, obj=None, **kwargs): + form = super().get_form(request, obj, **kwargs) + + # allow only superusers to modify Configuration + if not request.user.is_superuser: + for field in form.base_fields: + form.base_fields.get(field).disabled = True + + return form + + +class ThesaurusAdmin(admin.ModelAdmin): + change_list_template = "admin/thesauri/change_list.html" + + model = Thesaurus + list_display = ("id", "identifier") + list_display_links = ("id", "identifier") + ordering = ("identifier",) + + def get_urls(self): + urls = super().get_urls() + my_urls = [path("importrdf/", self.import_rdf, name="base_thesaurus_importrdf")] + return my_urls + urls + + def import_rdf(self, request): + if request.method == "POST": + try: + rdf_file = request.FILES["rdf_file"] + name = slugify(rdf_file.name) + call_command("load_thesaurus", file=rdf_file, name=name) + self.message_user(request, "Your RDF file has been imported", messages.SUCCESS) + return redirect("..") + except Exception as e: + self.message_user(request, e.args[0], messages.ERROR) + return redirect("..") + + form = ThesaurusImportForm() + payload = {"form": form} + return render(request, "admin/thesauri/upload_form.html", payload) + + +class ThesaurusLabelAdmin(admin.ModelAdmin): + model = ThesaurusLabel + list_display = ("thesaurus_id", "lang", "label") + list_display_links = ("label",) + ordering = ("thesaurus__identifier", "lang") + + def thesaurus_id(self, obj): + return obj.thesaurus.identifier + + thesaurus_id.short_description = "Thesaurus" + thesaurus_id.admin_order_field = "thesaurus__identifier" + + +class ThesaurusKeywordAdmin(admin.ModelAdmin): + model = ThesaurusKeyword + + list_display = ( + "thesaurus_id", + "about", + "alt_label", + ) + list_display_links = ( + "about", + "alt_label", + ) + ordering = ( + "thesaurus__identifier", + "alt_label", + ) + list_filter = ("thesaurus_id",) + + def thesaurus_id(self, obj): + return obj.thesaurus.identifier + + thesaurus_id.short_description = "Thesaurus" + thesaurus_id.admin_order_field = "thesaurus__identifier" + + +class ThesaurusKeywordLabelAdmin(admin.ModelAdmin): + model = ThesaurusKeywordLabel + + list_display = ("thesaurus_id", "keyword_id", "lang", "label") + list_display_links = ("lang", "label") + ordering = ("keyword__thesaurus__identifier", "keyword__alt_label", "lang") + list_filter = ("keyword__thesaurus__identifier", "keyword_id", "lang") + + def thesaurus_id(self, obj): + return obj.keyword.thesaurus.identifier + + thesaurus_id.short_description = "Thesaurus" + thesaurus_id.admin_order_field = "keyword__thesaurus__identifier" + + def keyword_id(self, obj): + return obj.keyword.alt_label + + keyword_id.short_description = "Keyword" + keyword_id.admin_order_field = "keyword__alt_label" + + +admin.site.register(TopicCategory, TopicCategoryAdmin) +admin.site.register(Region, RegionAdmin) +admin.site.register(SpatialRepresentationType, SpatialRepresentationTypeAdmin) +admin.site.register(RestrictionCodeType, RestrictionCodeTypeAdmin) +admin.site.register(ContactRole, ContactRoleAdmin) +admin.site.register(Link, LinkAdmin) +admin.site.register(License, LicenseAdmin) +admin.site.register(HierarchicalKeyword, HierarchicalKeywordAdmin) +admin.site.register(MenuPlaceholder, MenuPlaceholderAdmin) +admin.site.register(Menu, MenuAdmin) +admin.site.register(MenuItem, MenuItemAdmin) +admin.site.register(Configuration, ConfigurationAdmin) +admin.site.register(Thesaurus, ThesaurusAdmin) +admin.site.register(ThesaurusLabel, ThesaurusLabelAdmin) +admin.site.register(ThesaurusKeyword, ThesaurusKeywordAdmin) +admin.site.register(ThesaurusKeywordLabel, ThesaurusKeywordLabelAdmin) + + +class ResourceBaseAdminForm(autocomplete.FutureModelForm): + keywords = TagField(widget=TaggitSelect2Custom("autocomplete_hierachical_keyword"), required=False) + + def delete_queryset(self, request, queryset): + """ + We need to invoke the 'ResourceBase.delete' method even when deleting + through the admin batch action + """ + for obj in queryset: + from geonode.resource.manager import resource_manager + + resource_manager.delete(obj.uuid, instance=obj) + + class Meta: + pass diff --git a/geonode/base/api/__init__.py b/geonode/base/api/__init__.py new file mode 100644 index 0000000..0044807 --- /dev/null +++ b/geonode/base/api/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/base/api/exceptions.py b/geonode/base/api/exceptions.py new file mode 100644 index 0000000..3281229 --- /dev/null +++ b/geonode/base/api/exceptions.py @@ -0,0 +1,61 @@ +######################################################################### +# +# Copyright (C) 2022 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from rest_framework.exceptions import APIException +from rest_framework.views import exception_handler + + +class InvalidResourceException(APIException): + status_code = 500 + default_detail = "Invalid Resource exception" + default_code = "invalid_resource_exception" + category = "resource_api" + + +def geonode_exception_handler(exc, context): + # Call REST framework's default exception handler first, + # to get the standard error response. + response = exception_handler(exc, context) + + if response is not None and isinstance(exc, APIException): + # for the upload exception we need a custom response + detail = _extract_detail(exc) + response.data = { + "success": False, + "errors": [str(detail)], + "code": exc.code if hasattr(exc, "code") else exc.default_code, + } + return response + + +def _extract_detail(exc, loop=False): + if hasattr(exc, "detail"): + if isinstance(exc.detail, list): + return exc.detail[0] + elif isinstance(exc.detail, dict): + return _extract_detail(exc.detail, loop=True) + else: + return exc.detail + elif loop: + try: + error = exc.get(list(exc)[0]) + if isinstance(error, list): + return error[0] + except Exception: + return exc.default_detail + return exc.default_detail diff --git a/geonode/base/api/fields.py b/geonode/base/api/fields.py new file mode 100644 index 0000000..d719772 --- /dev/null +++ b/geonode/base/api/fields.py @@ -0,0 +1,54 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json + +from django.core.exceptions import ValidationError +from dynamic_rest.fields.fields import DynamicRelationField + + +class ComplexDynamicRelationField(DynamicRelationField): + def to_internal_value_single(self, data, serializer): + """Overwrite of DynamicRelationField implementation to handle complex data structure initialization + + Args: + data (Optional[str, Dict]}): serialized or deserialized data from http calls (POST, GET ...) + serializer (DynamicModelSerializer): Serializer for the given data + + Raises: + ValidationError: raised when requested data does not exist + + django.db.models.QuerySet: return QuerySet object of the request or set data + """ + related_model = serializer.Meta.model + if isinstance(data, str): + data = json.loads(data) + + if isinstance(data, dict): + try: + if hasattr(serializer, "many") and serializer.many is True: + return [serializer.get_model().objects.get(**d) for d in data] + return serializer.get_model().objects.get(**data) + except related_model.DoesNotExist: + raise ValidationError( + "Invalid value for '%s': %s object with ID=%s not found" + % (self.field_name, related_model.__name__, data) + ) + else: + return super().to_internal_value_single(data, serializer) diff --git a/geonode/base/api/filters.py b/geonode/base/api/filters.py new file mode 100644 index 0000000..a2428bf --- /dev/null +++ b/geonode/base/api/filters.py @@ -0,0 +1,133 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import ast +import logging +from distutils.util import strtobool +from itertools import groupby + +from rest_framework.filters import SearchFilter, BaseFilterBackend + +from django.db.models import Subquery + +from geonode.base.models import ThesaurusKeyword +from geonode.favorite.models import Favorite +from geonode.base.bbox_utils import filter_bbox + +logger = logging.getLogger(__name__) + + +class DynamicSearchFilter(SearchFilter): + def get_search_fields(self, view, request): + return request.GET.getlist("search_fields", []) + + +class ExtentFilter(BaseFilterBackend): + """ + Filter that only allows users to see their own objects. + """ + + def filter_queryset(self, request, queryset, view): + if request.query_params.get("extent"): + return filter_bbox(queryset, request.query_params.get("extent")) + return queryset + + +class TKeywordsFilter(BaseFilterBackend): + """ + TKeywords are a ManyToMany relation but DREST can't handle AND filtering the way we need. + When the filter has more than one tkeyword, DREST by default will return Resources associated at least to one + of the keywords. + """ + + def filter_queryset(self, request, queryset, view): + # we must make the GET mutable since in the filters, some queryparams are popped + request.GET._mutable = True + try: + return ( + self.filter_queryset_GROUP(request, queryset, view) + if "force_and" not in request.GET + else self.filter_queryset_AND(request, queryset, view) + ) + finally: + request.GET._mutable = False + + def filter_queryset_AND(self, request, queryset, view): + """ + This implementation requires all the tkeywords to be assigned to the Resource + """ + for v in request.GET.pop("filter{tkeywords}", []): + queryset = queryset.filter(tkeywords__id=v) + return queryset + + def filter_queryset_GROUP(self, request, queryset, view): + """ + This implementation requires that at least one tkeyword for each thesaurus is assigned to the Resource + """ + if tklist := request.GET.pop("filter{tkeywords}", None): + if len(tklist) == 1: + # if there's only one filtering keyword we don't need to tell to which thesaurus it belongs to + return queryset.filter(tkeywords__id=tklist[0]) + + tkinfo = ( + ThesaurusKeyword.objects.filter(id__in=tklist).values("id", "thesaurus__id").order_by("thesaurus__id") + ) + + for t, tk in groupby(tkinfo, lambda r: r["thesaurus__id"]): + tklist_by_t = [x["id"] for x in tk] + logger.info("Filtering by %s - %r keywords", t, tklist_by_t) + queryset = queryset.filter(tkeywords__id__in=tklist_by_t) + + return queryset + + +class FavoriteFilter(BaseFilterBackend): + """ + Filter that only allows users to see their own objects. + """ + + def filter_queryset(self, request, queryset, _): + if strtobool(request.query_params.get("favorite", "False")): + c_types = list({r.polymorphic_ctype.model for r in queryset}) + return queryset.filter( + pk__in=Subquery( + Favorite.objects.values_list("object_id", flat=True) + .filter(user=request.user) + .filter(content_type__model__in=c_types) + ) + ) + return queryset + + +class FacetVisibleResourceFilter(BaseFilterBackend): + """ + Return Only elements that have a resource assigned. + """ + + def filter_queryset(self, request, queryset, _): + _filter = {} + + _with_resources = ast.literal_eval(request.GET.get("with_resources", "False")) + + if _with_resources: + _filter["id__in"] = [_facet.id for _facet in queryset if _facet.resourcebase_set.exists()] + elif "with_resources" in request.GET and not _with_resources: + # check that the facet has been passed and is false + _filter["id__in"] = [_facet.id for _facet in queryset if not _facet.resourcebase_set.exists()] + + return queryset.filter(**_filter) diff --git a/geonode/base/api/mixins.py b/geonode/base/api/mixins.py new file mode 100644 index 0000000..9a641c4 --- /dev/null +++ b/geonode/base/api/mixins.py @@ -0,0 +1,55 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from distutils.util import strtobool +from rest_framework.mixins import ListModelMixin +from rest_framework.response import Response + + +class AdvertisedListMixin(ListModelMixin): + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + + # advertised + # if superuser, all resources will be visible, otherwise only the advertised one and + # the resource which the user is owner will be returned + user = request.user + try: + _filter = request.query_params.get("advertised", "None") + advertised = strtobool(_filter) if _filter.lower() != "all" else "all" + except Exception: + advertised = None + + if advertised == "all": + pass + elif advertised is not None: + queryset = queryset.filter(advertised=advertised) + else: + is_admin = user.is_superuser if user and user.is_authenticated else False + if not is_admin and user and not user.is_anonymous: + queryset = (queryset.filter(advertised=True) | queryset.filter(owner=user)).distinct() + elif not user or user.is_anonymous: + queryset = queryset.filter(advertised=True) + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) diff --git a/geonode/base/api/pagination.py b/geonode/base/api/pagination.py new file mode 100644 index 0000000..dd4b545 --- /dev/null +++ b/geonode/base/api/pagination.py @@ -0,0 +1,41 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.conf import settings +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination + +DEFAULT_PAGE = getattr(settings, "REST_API_DEFAULT_PAGE", 1) +DEFAULT_PAGE_SIZE = getattr(settings, "REST_API_DEFAULT_PAGE_SIZE", 10) +DEFAULT_PAGE_QUERY_PARAM = getattr(settings, "REST_API_DEFAULT_PAGE_QUERY_PARAM", "page_size") + + +class GeoNodeApiPagination(PageNumberPagination): + page = DEFAULT_PAGE + page_size = DEFAULT_PAGE_SIZE + page_size_query_param = DEFAULT_PAGE_QUERY_PARAM + + def get_paginated_response(self, data): + _paginated_response = { + "links": {"next": self.get_next_link(), "previous": self.get_previous_link()}, + "total": self.page.paginator.count, + "page": int(self.request.GET.get("page", DEFAULT_PAGE)), # can not set default = self.page + DEFAULT_PAGE_QUERY_PARAM: int(self.request.GET.get(DEFAULT_PAGE_QUERY_PARAM, self.page_size)), + } + _paginated_response.update(data) + return Response(_paginated_response) diff --git a/geonode/base/api/permissions.py b/geonode/base/api/permissions.py new file mode 100644 index 0000000..c5c3653 --- /dev/null +++ b/geonode/base/api/permissions.py @@ -0,0 +1,287 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +from django.conf import settings +from django.contrib.auth import get_user_model +from django.shortcuts import get_object_or_404 + +from rest_framework import permissions +from rest_framework.filters import BaseFilterBackend +from geonode.people.utils import get_available_users +from geonode.security.permissions import ( + BASIC_MANAGE_PERMISSIONS, + DOWNLOAD_PERMISSIONS, + EDIT_PERMISSIONS, + VIEW_PERMISSIONS, +) +from distutils.util import strtobool +from geonode.security.utils import get_users_with_perms, get_visible_resources +from geonode.groups.models import GroupProfile +from rest_framework.permissions import DjangoModelPermissions +from guardian.shortcuts import get_objects_for_user +from itertools import chain +from guardian.shortcuts import get_groups_with_perms + +logger = logging.getLogger(__name__) + + +class IsSelf(permissions.BasePermission): + """Grant permission only if the current instance is the request user. + Used to allow users to edit their own account, nothing to others (even + superusers). + """ + + def has_permission(self, request, view): + """Always return False here. + The fine-grained permissions are handled in has_object_permission(). + """ + return False + + def has_object_permission(self, request, view, obj): + user = request.user + if user and isinstance(obj, get_user_model()) and obj.pk == user.pk: + return True + + return False + + +class IsSelfOrReadOnly(IsSelf): + """Grant permissions if instance *IS* the request user, or read-only. + Used to allow users to edit their own account, and others to read. + """ + + def has_object_permission(self, request, view, obj): + if request.method in permissions.SAFE_METHODS: + return True + + return IsSelf.has_object_permission(self, request, view, obj) + + +class IsSelfOrAdmin(IsSelf): + """Grant R/W to self and superusers/staff members. Deny others.""" + + def has_permission(self, request, view): + user = request.user + if user and (user.is_superuser or user.is_staff): + return True + + return IsSelf.has_permission(self, request, view) + + def has_object_permission(self, request, view, obj): + user = request.user + if user and (user.is_superuser or user.is_staff): + return True + + return IsSelf.has_object_permission(self, request, view, obj) + + +class IsSelfOrAdminOrReadOnly(IsSelfOrAdmin): + """Grant R/W to self and superusers/staff members, R/O to others.""" + + def has_permission(self, request, view): + if request.method in permissions.SAFE_METHODS: + return True + + return IsSelfOrAdmin.has_permission(self, request, view) + + def has_object_permission(self, request, view, obj): + if request.method in permissions.SAFE_METHODS: + return True + + return IsSelfOrAdmin.has_object_permission(self, request, view, obj) + + +class IsSelfOrAdminOrAuthenticatedReadOnly(IsSelfOrAdmin): + """Grant R/W to self and superusers/staff members, R/O to auth.""" + + def has_object_permission(self, request, view, obj): + user = request.user + if request.method in permissions.SAFE_METHODS: + if user.is_authenticated(): + return True + + return IsSelfOrAdmin.has_object_permission(self, request, view, obj) + + +class IsOwnerOrAdmin(permissions.BasePermission): + """ + Object-level permission to only allow admin and owners of an object to edit it. + Assumes the model instance has an `owner` attribute. + """ + + def has_object_permission(self, request, view, obj): + if request.user is None or (not request.user.is_anonymous and not request.user.is_active): + return False + if request.user.is_superuser or request.user.is_staff: + return True + + # Instance must have an attribute named `owner`. + _request_matches = False + if isinstance(obj, get_user_model()) and obj == request.user: + _request_matches = True + elif hasattr(obj, "owner"): + _request_matches = obj.owner == request.user + elif hasattr(obj, "user"): + _request_matches = obj.user == request.user + + if isinstance(obj, get_user_model()) and not request.user.is_anonymous: + if request.method in permissions.SAFE_METHODS and obj in get_available_users(request.user): + return True + return _request_matches + + if not _request_matches: + _request_matches = request.user in get_users_with_perms(obj) + return _request_matches + + +class IsOwnerOrReadOnly(IsOwnerOrAdmin): + """ + Object-level permission to only allow owners of an object to edit it. + Assumes the model instance has an `owner` attribute. + """ + + def has_object_permission(self, request, view, obj): + # Read permissions are allowed to any request, + # so we'll always allow GET, HEAD or OPTIONS requests. + if request.method in permissions.SAFE_METHODS and not isinstance(obj, get_user_model()): + return True + + return IsOwnerOrAdmin.has_object_permission(self, request, view, obj) + + +class IsManagerEditOrAdmin(permissions.BasePermission): + """ + Object-level permission to only allow admin and managers to edit a group. + """ + + def has_permission(self, request, view): + if request.method in ["POST", "DELETE"]: + user = request.user + return user and (user.is_superuser or user.is_staff) + + return True + + def has_object_permission(self, request, view, obj): + # Read permissions are allowed to any request, + # so we'll always allow GET, HEAD or OPTIONS requests + if request.method in permissions.SAFE_METHODS: + return True + + user = request.user + if user and user.is_superuser or user.is_staff: + return True + + is_group_manager = user and isinstance(obj, GroupProfile) and obj.user_is_role(user, "manager") + if is_group_manager and request.method == "PATCH": + return True + + return False + + +class ResourceBasePermissionsFilter(BaseFilterBackend): + """ + A filter backend that limits results to those where the requesting user + has read object level permissions. + """ + + def filter_queryset(self, request, queryset, view): + try: + metadata_only = strtobool(request.query_params.get("filter{metadata_only}", "None")) + except Exception: + metadata_only = None + + return get_visible_resources( + queryset, + request.user, + metadata_only=metadata_only, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + +class UserHasPerms(DjangoModelPermissions): + perms_map = { + "GET": [f"base.{x}" for x in VIEW_PERMISSIONS + DOWNLOAD_PERMISSIONS], + "POST": ["base.add_resourcebase"] + [f"base.{x}" for x in EDIT_PERMISSIONS], + "PUT": [f"base.{x}" for x in EDIT_PERMISSIONS], + "PATCH": [f"base.{x}" for x in EDIT_PERMISSIONS], + "DELETE": [f"base.{x}" for x in BASIC_MANAGE_PERMISSIONS], + } + + def __init__(self, perms_dict={}): + self.perms_dict = perms_dict + + def __call__(self): + return self + + def has_permission(self, request, view): + from geonode.base.models import ResourceBase + + queryset = self._queryset(view) + + if request.user.is_superuser: + return True + + if view.kwargs.get("pk"): + # if a single resource is called, we check the perms for that resource + res = get_object_or_404(ResourceBase, pk=view.kwargs.get("pk")) + # if the request is for a single resource, we take the specific or the default. If none is defined we keep the original one defined above + resource_type_specific_perms = self.perms_dict.get( + res.get_real_instance().resource_type, self.perms_dict.get("default", {}) + ) + perms = resource_type_specific_perms.get(request.method, []) or self.get_required_permissions( + request.method, queryset.model + ) + + # getting the user permission for that resource + resource_perms = res.get_user_perms(request.user) + + groups = get_groups_with_perms(res, attach_perms=True) + # we are making this because the request.user.groups sometimes returns empty si is not fully reliable + for group, perm in groups.items(): + # checking if the user is in that group + if group.user_set.filter(username=request.user).exists(): + resource_perms = list(chain(resource_perms, perm)) + + if request.user.has_perm("base.add_resourcebase"): + resource_perms.append("add_resourcebase") + # merging all available permissions into a single list + available_perms = list(set(resource_perms)) + # fixup the permissions name + perms_without_base = [x.replace("base.", "") for x in perms] + # if at least one of the permissions is available the request is True + rule = resource_type_specific_perms.get("rule", any) + return rule([_perm in available_perms for _perm in perms_without_base]) + + if request.method in permissions.SAFE_METHODS: + return True + + _default_defined_perms = self.perms_dict.get("default", {}) + if _default_defined_perms.get(request.method): + _defined_perms = _default_defined_perms.get(request.method) + rule = _default_defined_perms.get("rule", any) + return rule([request.user.has_perm(_perm) for _perm in _defined_perms]) + + perms = self.perms_dict.get(request.method, None) or self.get_required_permissions( + request.method, queryset.model + ) + + # check if the user have one of the perms in all the resource available + return get_objects_for_user(request.user, perms).exists() diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py new file mode 100644 index 0000000..9a5ad98 --- /dev/null +++ b/geonode/base/api/serializers.py @@ -0,0 +1,878 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from slugify import slugify +from urllib.parse import urljoin +import json + +from django.db.models import Q +from django.conf import settings +from django.contrib.auth.models import Group +from django.forms.models import model_to_dict +from django.contrib.auth import get_user_model +from django.db.models.query import QuerySet +from geonode.assets.utils import get_default_asset +from geonode.people import Roles +from django.http import QueryDict +from deprecated import deprecated +from rest_framework import serializers +from rest_framework_gis import fields +from rest_framework.reverse import reverse, NoReverseMatch +from rest_framework.exceptions import ParseError + +from dynamic_rest.serializers import DynamicEphemeralSerializer, DynamicModelSerializer +from dynamic_rest.fields.fields import DynamicRelationField, DynamicComputedField + +from avatar.templatetags.avatar_tags import avatar_url +from geonode.utils import bbox_swap +from geonode.base.api.exceptions import InvalidResourceException + +from geonode.favorite.models import Favorite +from geonode.base.models import ( + Link, + ResourceBase, + HierarchicalKeyword, + Region, + RestrictionCodeType, + License, + TopicCategory, + SpatialRepresentationType, + ThesaurusKeyword, + ThesaurusKeywordLabel, + ExtraMetadata, + LinkedResource, +) +from geonode.documents.models import Document +from geonode.geoapps.models import GeoApp +from geonode.groups.models import GroupCategory, GroupProfile +from geonode.base.api.fields import ComplexDynamicRelationField +from geonode.layers.utils import get_download_handlers, get_default_dataset_download_handler +from geonode.assets.handlers import asset_handler_registry +from geonode.utils import build_absolute_uri +from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes +from geonode.resource.models import ExecutionRequest +from django.contrib.gis.geos import Polygon + +logger = logging.getLogger(__name__) + + +def user_serializer(): + import geonode.people.api.serializers as ser + + return ser.UserSerializer + + +class BaseDynamicModelSerializer(DynamicModelSerializer): + def to_representation(self, instance): + data = super().to_representation(instance) + if not isinstance(data, int): + try: + path = reverse(self.Meta.view_name) + if not path.endswith("/"): + path = f"{path}/" + url = urljoin(path, str(instance.pk)) + data["link"] = build_absolute_uri(url) + + parents = [] + parent = self.parent + while parent: + parents.append(type(parent).__name__) + parent = parent.parent + + logger.warning( + f"Deprecated: BaseDynamicModelSerializer should be replaced with proper Field" + f" - Parents: {parents} Root: {type(self).__name__}" + ) + except (TypeError, NoReverseMatch) as e: + logger.exception(e) + return data + + +class ResourceBaseTypesSerializer(DynamicEphemeralSerializer): + name = serializers.CharField() + count = serializers.IntegerField() + + class Meta: + name = "resource-types" + + +class PermSpecSerialiazer(DynamicEphemeralSerializer): + class Meta: + name = "perm-spec" + + class PermSpecFieldSerialiazer(DynamicEphemeralSerializer): + perm_spec = serializers.ListField() + + users = PermSpecFieldSerialiazer(many=True) + groups = PermSpecFieldSerialiazer(many=True) + + +class GroupSerializer(DynamicModelSerializer): + class Meta: + model = Group + name = "group" + fields = ("pk", "name") + + +class GroupProfileSerializer(BaseDynamicModelSerializer): + class Meta: + model = GroupProfile + name = "group_profile" + view_name = "group-profiles-list" + fields = ("pk", "title", "group", "slug", "logo", "description", "email", "keywords", "access", "categories") + + group = DynamicRelationField(GroupSerializer, embed=True, many=False) + keywords = serializers.SlugRelatedField(many=True, slug_field="slug", read_only=True) + categories = serializers.SlugRelatedField(many=True, slug_field="slug", queryset=GroupCategory.objects.all()) + + +class SimpleHierarchicalKeywordSerializer(DynamicModelSerializer): + class Meta: + model = HierarchicalKeyword + name = "HierarchicalKeyword" + fields = ("name", "slug") + + def to_representation(self, value): + return {"name": value.name, "slug": value.slug} + + +class _ThesaurusKeywordSerializerMixIn: + def to_representation(self, value): + _i18n = {} + for _i18n_label in ThesaurusKeywordLabel.objects.filter(keyword__id=value.id).iterator(): + _i18n[_i18n_label.lang] = _i18n_label.label + return { + "name": value.alt_label, + "slug": slugify(value.about), + "uri": value.about, + "thesaurus": { + "name": value.thesaurus.title, + "slug": value.thesaurus.identifier, + "uri": value.thesaurus.about, + }, + "i18n": _i18n, + } + + +class SimpleThesaurusKeywordSerializer(_ThesaurusKeywordSerializerMixIn, DynamicModelSerializer): + class Meta: + model = ThesaurusKeyword + name = "ThesaurusKeyword" + fields = ("alt_label",) + + +class SimpleRegionSerializer(DynamicModelSerializer): + class Meta: + model = Region + name = "Region" + fields = ("code", "name") + + +class SimpleTopicCategorySerializer(DynamicModelSerializer): + class Meta: + model = TopicCategory + name = "TopicCategory" + fields = ("identifier", "gn_description") + + +class RestrictionCodeTypeSerializer(DynamicModelSerializer): + class Meta: + model = RestrictionCodeType + name = "RestrictionCodeType" + fields = ("identifier",) + + +class LicenseSerializer(DynamicModelSerializer): + class Meta: + model = License + name = "License" + fields = ("identifier",) + + +class SpatialRepresentationTypeSerializer(DynamicModelSerializer): + class Meta: + model = SpatialRepresentationType + name = "SpatialRepresentationType" + fields = ("identifier",) + + +class AvatarUrlField(DynamicComputedField): + def __init__(self, avatar_size, **kwargs): + self.avatar_size = avatar_size + super().__init__(**kwargs) + + def get_attribute(self, instance): + return build_absolute_uri(avatar_url(instance, self.avatar_size)) + + +class EmbedUrlField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + try: + _instance = instance.get_real_instance() + except Exception as e: + logger.exception(e) + _instance = None + if _instance and hasattr(_instance, "embed_url") and _instance.embed_url != NotImplemented: + return build_absolute_uri(_instance.embed_url) + else: + return "" + + +class DetailUrlField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + return build_absolute_uri(instance.detail_url) + + +class ExtraMetadataSerializer(DynamicModelSerializer): + class Meta: + model = ExtraMetadata + name = "ExtraMetadata" + fields = ("pk", "metadata") + + def to_representation(self, obj): + if isinstance(obj, QuerySet): + out = [] + for el in obj: + out.append({**{"id": el.id}, **el.metadata}) + return out + elif isinstance(obj, list): + return obj + return {**{"id": obj.id}, **obj.metadata} + + +class ThumbnailUrlField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + thumbnail_url = instance.thumbnail_url + + return build_absolute_uri(thumbnail_url) + + +class DownloadLinkField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @deprecated(version="4.2.0", reason="Will be replaced by download_urls") + def get_attribute(self, instance): + try: + logger.info( + f"Field {self.field_name} is deprecated and will be removed in the future GeoNode version. Please refer to download_urls" + ) + _instance = instance.get_real_instance() + return _instance.download_url if hasattr(_instance, "download_url") else None + except Exception as e: + logger.exception(e) + return None + + +class DownloadArrayLinkField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + try: + _instance = instance.get_real_instance() + except Exception as e: + logger.exception(e) + raise e + + asset = get_default_asset(_instance) + if asset is not None: + asset_url = asset_handler_registry.get_handler(asset).create_download_url(asset) + + if _instance.resource_type in ["map"] + get_geoapp_subtypes(): + return [] + elif _instance.resource_type in ["document"]: + payload = [ + { + "url": _instance.download_url, + "ajax_safe": _instance.download_is_ajax_safe, + }, + ] + if asset: + payload.append({"url": asset_url, "ajax_safe": False, "default": False}) + return payload + + elif _instance.resource_type in ["dataset"]: + download_urls = [] + # lets get only the default one first to set it + default_handler = get_default_dataset_download_handler() + obj = default_handler(self.context.get("request"), _instance.alternate) + if obj.download_url: + download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": True}) + # then let's prepare the payload with everything + for handler in get_download_handlers(): + obj = handler(self.context.get("request"), _instance.alternate) + if obj.download_url: + download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": False}) + + if asset: + download_urls.append({"url": asset_url, "ajax_safe": True, "default": False if download_urls else True}) + + return download_urls + else: + return [] + + +class FavoriteField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + _user = self.context.get("request") + if _user and not _user.user.is_anonymous: + return Favorite.objects.filter(object_id=instance.pk, user=_user.user).exists() + return False + + +class AutoLinkField(DynamicComputedField): + + def get_attribute(self, instance): + parents = [] + parent = self.parent + while parent: + parents.append(type(parent).__name__) + parent = parent.parent + + logger.debug( + f"AutoLinkField reading Meta from first parent - Parents: {parents} root: {type(self.root).__name__}" + ) + + try: + path = reverse(self.parent.Meta.view_name) + if not path.endswith("/"): + path = f"{path}/" + url = urljoin(path, str(instance.pk)) + return build_absolute_uri(url) + + except AttributeError as e: + logger.exception(f"Parents: {parents} root: {type(self.root).__name__}", exc_info=e) + return None + + except Exception as e: + logger.exception(e) + return None + + +class ContactRoleField(DynamicComputedField): + default_error_messages = { + "required": ("ContactRoleField This field is required."), + } + + def __init__(self, contact_type, **kwargs): + self.contact_type = contact_type + super().__init__(**kwargs) + + def get_attribute(self, instance): + return getattr(instance, self.contact_type) + + def to_representation(self, value): + return [user_serializer()(embed=True, many=False).to_representation(v) for v in value] + + def get_pks_of_users_to_set(self, value): + pks_of_users_to_set = [] + for val in value: + # make it possible to set contact roles via username or pk through API + if "username" in val and "pk" in val: + pk = val["pk"] + username = val["username"] + pk_user = get_user_model().objects.get(pk=pk) + username_user = get_user_model().objects.get(username=username) + if pk_user.pk != username_user.pk: + raise ParseError( + detail=f"user with pk: {pk} and username: {username} is not the same ... ", code=403 + ) + pks_of_users_to_set.append(pk) + elif "username" in val: + username = val["username"] + username_user = get_user_model().objects.get(username=[username]) + pks_of_users_to_set.append(username_user.pk) + elif "pk" in val: + pks_of_users_to_set.append(val["pk"]) + return pks_of_users_to_set + + def to_internal_value(self, value): + return get_user_model().objects.filter(pk__in=self.get_pks_of_users_to_set(value)) + + +class ExtentBboxField(DynamicComputedField): + def get_attribute(self, instance): + return instance.ll_bbox + + def to_representation(self, value): + bbox = bbox_swap(value[:-1]) + return super().to_representation({"coords": bbox, "srid": value[-1]}) + + +class DataBlobField(DynamicRelationField): + def value_to_string(self, obj): + value = self.value_from_object(obj) + return self.get_prep_value(value) + + +class DataBlobSerializer(DynamicModelSerializer): + class Meta: + model = ResourceBase + fields = ("pk", "blob") + + def to_representation(self, value): + data = ResourceBase.objects.filter(id=value) + if data.exists() and data.count() == 1: + return data.get().blob + return {} + + +class ResourceExecutionRequestSerializer(DynamicModelSerializer): + class Meta: + model = ResourceBase + fields = ("pk",) + + def to_representation(self, instance): + data = [] + request = self.context.get("request", None) + if ( + request + and request.user + and not request.user.is_anonymous + and ResourceBase.objects.filter(pk=instance).count() == 1 + ): + _resource = ResourceBase.objects.get(pk=instance) + executions = ExecutionRequest.objects.filter( + Q(user=request.user) + & ~Q(status=ExecutionRequest.STATUS_FINISHED) + & ( + ( + Q(input_params__uuid=_resource.uuid) + | Q(output_params__output__uuid=_resource.uuid) + | Q(geonode_resource=_resource) + ) + ) + ).order_by("-created") + + for execution in executions: + data.append( + { + "exec_id": execution.exec_id, + "user": execution.user.username, + "status": execution.status, + "func_name": execution.func_name, + "created": execution.created, + "finished": execution.finished, + "last_updated": execution.last_updated, + "input_params": execution.input_params, + "output_params": execution.output_params, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": execution.exec_id}) + ), + }, + ) + return data + + +class LinkedResourceEmbeddedSerializer(DynamicModelSerializer): + class Meta: + model = ResourceBase + fields = ("pk",) + + def to_representation(self, instance): + from geonode.base.api.views import base_linked_resources_payload + + request = self.context.get("request", None) + _resource = ResourceBase.objects.get(pk=instance) + + return ( + base_linked_resources_payload(_resource.get_real_instance(), request.user) + if request and request.user and _resource + else {} + ) + + +api_bbox_settable_resource_models = [Document, GeoApp] + + +class PermsSerializer(DynamicModelSerializer): + class Meta: + model = ResourceBase + fields = ("pk",) + + def to_representation(self, instance): + request = self.context.get("request", None) + resource = ResourceBase.objects.get(pk=instance) + return resource.get_user_perms(request.user) if request and request.user and resource else [] + + +class LinksSerializer(DynamicModelSerializer): + class Meta: + model = ResourceBase + + def to_representation(self, instance): + ret = [] + link_fields = ["extension", "link_type", "name", "mime", "url"] + links = Link.objects.filter( + resource_id=instance, # link_type__in=["OGC:WMS", "OGC:WFS", "OGC:WCS", "image", "metadata"] + ) + for lnk in links: + formatted_link = model_to_dict(lnk, fields=link_fields) + ret.append(formatted_link) + if lnk.asset: + extras = { + "type": "asset", + "content": model_to_dict(lnk.asset, ["title", "description", "type", "created"]), + } + extras["content"]["download_url"] = asset_handler_registry.get_handler(lnk.asset).create_download_url( + lnk.asset + ) + formatted_link["extras"] = extras + + return ret + + +class ResourceManagementField(serializers.BooleanField): + MAPPING = {"is_approved": "can_approve", "is_published": "can_publish", "featured": "can_feature"} + + def to_internal_value(self, data): + new_val = super().to_internal_value(data) + user = self.context["request"].user + user_action = self.MAPPING.get(self.field_name) + instance = self.root.instance or ResourceBase.objects.get(pk=self.root.initial_data["pk"]) + if getattr(user, user_action)(instance): + logger.debug("User can perform the action, the new value is returned") + return new_val + else: + logger.warning(f"The user does not have the perms to update the value of {self.field_name}") + return getattr(instance, self.field_name) + + +class ResourceBaseSerializer(DynamicModelSerializer): + pk = serializers.CharField(read_only=True) + uuid = serializers.CharField(read_only=True) + resource_type = serializers.CharField(required=False) + polymorphic_ctype_id = serializers.CharField(read_only=True) + owner = DynamicRelationField(user_serializer(), embed=True, read_only=True) + metadata_author = ContactRoleField(Roles.METADATA_AUTHOR.name, required=False) + processor = ContactRoleField(Roles.PROCESSOR.name, required=False) + publisher = ContactRoleField(Roles.PUBLISHER.name, required=False) + custodian = ContactRoleField(Roles.CUSTODIAN.name, required=False) + poc = ContactRoleField(Roles.POC.name, required=False) + distributor = ContactRoleField(Roles.DISTRIBUTOR.name, required=False) + resource_user = ContactRoleField(Roles.RESOURCE_USER.name, required=False) + resource_provider = ContactRoleField(Roles.RESOURCE_PROVIDER.name, required=False) + originator = ContactRoleField(Roles.ORIGINATOR.name, required=False) + principal_investigator = ContactRoleField(Roles.PRINCIPAL_INVESTIGATOR.name, required=False) + title = serializers.CharField(required=False) + abstract = serializers.CharField(required=False) + attribution = serializers.CharField(required=False) + doi = serializers.CharField(required=False) + alternate = serializers.CharField(read_only=True, required=False) + date = serializers.DateTimeField(required=False) + date_type = serializers.CharField(required=False) + temporal_extent_start = serializers.DateTimeField(required=False) + temporal_extent_end = serializers.DateTimeField(required=False) + edition = serializers.CharField(required=False) + purpose = serializers.CharField(required=False) + maintenance_frequency = serializers.CharField(required=False) + constraints_other = serializers.CharField(required=False) + language = serializers.CharField(required=False) + supplemental_information = serializers.CharField(required=False) + data_quality_statement = serializers.CharField(required=False) + bbox_polygon = fields.GeometryField(read_only=True, required=False) + ll_bbox_polygon = fields.GeometryField(read_only=True, required=False) + extent = ExtentBboxField(required=False) + srid = serializers.CharField(required=False) + group = ComplexDynamicRelationField(GroupSerializer, embed=True) + popular_count = serializers.CharField(required=False) + share_count = serializers.CharField(required=False) + rating = serializers.CharField(required=False) + featured = ResourceManagementField(required=False) + advertised = serializers.BooleanField(required=False) + is_published = ResourceManagementField(required=False) + is_approved = ResourceManagementField(required=False) + detail_url = DetailUrlField(read_only=True) + created = serializers.DateTimeField(read_only=True) + last_updated = serializers.DateTimeField(read_only=True) + raw_abstract = serializers.CharField(read_only=True) + raw_purpose = serializers.CharField(read_only=True) + raw_constraints_other = serializers.CharField(read_only=True) + raw_supplemental_information = serializers.CharField(read_only=True) + raw_data_quality_statement = serializers.CharField(read_only=True) + metadata_only = serializers.BooleanField(required=False) + processed = serializers.BooleanField(read_only=True) + state = serializers.CharField(read_only=True) + sourcetype = serializers.CharField(read_only=True) + embed_url = EmbedUrlField(required=False) + thumbnail_url = ThumbnailUrlField(read_only=True) + keywords = ComplexDynamicRelationField(SimpleHierarchicalKeywordSerializer, many=True) + tkeywords = ComplexDynamicRelationField(SimpleThesaurusKeywordSerializer, many=True) + regions = DynamicRelationField(SimpleRegionSerializer, embed=True, many=True, read_only=True) + category = ComplexDynamicRelationField(SimpleTopicCategorySerializer, embed=True) + restriction_code_type = ComplexDynamicRelationField(RestrictionCodeTypeSerializer, embed=True) + license = ComplexDynamicRelationField(LicenseSerializer, embed=True) + spatial_representation_type = ComplexDynamicRelationField(SpatialRepresentationTypeSerializer, embed=True) + blob = serializers.JSONField(required=False, write_only=True) + is_copyable = serializers.BooleanField(read_only=True) + download_url = DownloadLinkField(read_only=True) + favorite = FavoriteField(read_only=True) + download_urls = DownloadArrayLinkField(read_only=True) + perms = DynamicRelationField(PermsSerializer, source="id", read_only=True) + links = DynamicRelationField(LinksSerializer, source="id", read_only=True) + + # Deferred fields + metadata = ComplexDynamicRelationField(ExtraMetadataSerializer, many=True, deferred=True) + data = DataBlobField(DataBlobSerializer, source="id", deferred=True, required=False) + executions = DynamicRelationField( + ResourceExecutionRequestSerializer, source="id", deferred=True, required=False, read_only=True + ) + linked_resources = DynamicRelationField( + LinkedResourceEmbeddedSerializer, source="id", deferred=True, required=False, read_only=True + ) + link = AutoLinkField(read_only=True) + + class Meta: + model = ResourceBase + name = "resource" + view_name = "base-resources-list" + fields = ( + "pk", + "uuid", + "resource_type", + "polymorphic_ctype_id", + "perms", + "owner", + "poc", + "metadata_author", + "processor", + "publisher", + "custodian", + "distributor", + "resource_user", + "resource_provider", + "originator", + "principal_investigator", + "keywords", + "tkeywords", + "regions", + "category", + "title", + "abstract", + "attribution", + "alternate", + "doi", + "bbox_polygon", + "ll_bbox_polygon", + "srid", + "date", + "date_type", + "edition", + "purpose", + "maintenance_frequency", + "restriction_code_type", + "constraints_other", + "license", + "language", + "spatial_representation_type", + "temporal_extent_start", + "temporal_extent_end", + "supplemental_information", + "data_quality_statement", + "group", + "popular_count", + "share_count", + "rating", + "featured", + "advertised", + "is_published", + "is_approved", + "detail_url", + "embed_url", + "created", + "last_updated", + "raw_abstract", + "raw_purpose", + "raw_constraints_other", + "raw_supplemental_information", + "raw_data_quality_statement", + "metadata_only", + "processed", + "state", + "data", + "subtype", + "sourcetype", + "is_copyable", + "blob", + "metadata", + "executions", + "linked_resources", + "download_url", + "download_urls", + "extent", + "favorite", + "thumbnail_url", + "links", + "link", + "metadata_uploaded_preserve", + # TODO + # csw_typename, csw_schema, csw_mdsource, csw_insert_date, csw_type, csw_anytext, csw_wkt_geometry, + # metadata_uploaded, metadata_uploaded_preserve, metadata_xml, + # users_geolimits, groups_geolimits + ) + + def to_internal_value(self, data): + if isinstance(data, str): + data = json.loads(data) + if "data" in data: + data["blob"] = data.pop("data") + if isinstance(data, QueryDict): + data = data.dict() + data = super(ResourceBaseSerializer, self).to_internal_value(data) + return data + + def save(self, **kwargs): + extent = self.validated_data.pop("extent", None) + instance = super().save(**kwargs) + if extent and instance.get_real_instance()._meta.model in api_bbox_settable_resource_models: + srid = extent.get("srid", "EPSG:4326") + coords = extent.get("coords") + if not coords: + logger.warning("BBOX was sent, but no coords were supplied. Skipping") + return instance + try: + # small validation test + Polygon.from_bbox(coords) + except Exception as e: + logger.exception(e) + raise InvalidResourceException("The standard bbox provided is invalid") + instance.set_bbox_polygon(coords, srid) + return instance + + +class FavoriteSerializer(DynamicModelSerializer): + resource = serializers.SerializerMethodField() + + class Meta: + model = Favorite + name = "favorites" + fields = ("resource",) + + def to_representation(self, value): + data = super().to_representation(value) + return data["resource"] + + def get_resource(self, instance): + resource = ResourceBase.objects.get(pk=instance.object_id) + return ResourceBaseSerializer(resource).data + + +class BaseResourceCountSerializer(BaseDynamicModelSerializer): + def to_representation(self, instance): + request = self.context.get("request") + filter_options = {} + if request.query_params: + filter_options = { + "type_filter": request.query_params.get("type"), + "title_filter": request.query_params.get("title__icontains"), + } + data = super().to_representation(instance) + if not isinstance(data, int): + try: + count_filter = {self.Meta.count_type: instance} + data["count"] = get_resources_with_perms(request.user, filter_options).filter(**count_filter).count() + except (TypeError, NoReverseMatch) as e: + logger.exception(e) + return data + + +class HierarchicalKeywordSerializer(BaseResourceCountSerializer): + class Meta: + name = "keywords" + model = HierarchicalKeyword + count_type = "keywords" + view_name = "keywords-list" + fields = "__all__" + + +class ThesaurusKeywordSerializer(_ThesaurusKeywordSerializerMixIn, BaseResourceCountSerializer): + class Meta: + model = ThesaurusKeyword + name = "tkeywords" + view_name = "tkeywords-list" + count_type = "tkeywords" + fields = "__all__" + + +class RegionSerializer(BaseResourceCountSerializer): + class Meta: + name = "regions" + model = Region + count_type = "regions" + view_name = "regions-list" + fields = "__all__" + + +class TopicCategorySerializer(BaseResourceCountSerializer): + class Meta: + name = "categories" + model = TopicCategory + count_type = "category" + view_name = "categories-list" + fields = "__all__" + + +class OwnerSerializer(BaseResourceCountSerializer): + class Meta: + name = "owners" + count_type = "owner" + view_name = "owners-list" + model = get_user_model() + fields = ("pk", "username", "first_name", "last_name", "avatar", "perms") + + avatar = AvatarUrlField(240, read_only=True) + + +class LinkedResourceSerializer(DynamicModelSerializer): + def __init__(self, *kargs, serialize_source: bool = False, **kwargs): + super().__init__(*kargs, **kwargs) + self.serialize_target = not serialize_source + + class Meta: + name = "linked_resources" + model = LinkedResource + fields = ("internal",) + + def to_representation(self, instance: LinkedResource): + data = super().to_representation(instance) + item: ResourceBase = instance.target if self.serialize_target else instance.source + data.update( + { + "pk": item.pk, + "title": item.title, + "resource_type": item.resource_type, + "detail_url": item.detail_url, + "thumbnail_url": item.thumbnail_url, + } + ) + return data diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py new file mode 100644 index 0000000..5ef4047 --- /dev/null +++ b/geonode/base/api/tests.py @@ -0,0 +1,3537 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import ast +import os +import re +import sys +import json +import logging +from builtins import Exception +from typing import Iterable + +from django.test import RequestFactory, override_settings +import gisdata +from PIL import Image +from io import BytesIO +from time import sleep +from uuid import uuid4 +from unittest.mock import patch +from urllib.parse import urljoin +from datetime import date, timedelta + +from django.urls import reverse +from django.conf import settings +from django.core.files.uploadedfile import SimpleUploadedFile +from django.contrib.auth import get_user_model + +from owslib.etree import etree +from avatar.templatetags.avatar_tags import avatar_url + +from rest_framework.test import APITestCase +from rest_framework.renderers import JSONRenderer +from rest_framework.parsers import JSONParser + +from geonode.catalogue import get_catalogue +from geonode.catalogue.models import catalogue_post_save +from geonode.catalogue.views import csw_global_dispatch + +from geonode.resource.manager import resource_manager +from guardian.shortcuts import get_anonymous_user + +from geonode.assets.utils import create_asset_and_link +from geonode.maps.models import Map, MapLayer +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.assets.utils import get_default_asset +from geonode.assets.handlers import asset_handler_registry + +from geonode.base import enumerations +from geonode.base.api.serializers import ResourceBaseSerializer +from geonode.groups.models import GroupMember, GroupProfile +from geonode.thumbs.exceptions import ThumbnailError +from geonode.layers.utils import get_files +from geonode.base.models import ( + HierarchicalKeyword, + Region, + ResourceBase, + TopicCategory, + ThesaurusKeyword, + ExtraMetadata, + RestrictionCodeType, + License, + Group, + LinkedResource, +) + +from geonode.layers.models import Dataset +from geonode.favorite.models import Favorite +from geonode.documents.models import Document +from geonode.geoapps.models import GeoApp +from geonode.utils import build_absolute_uri +from geonode.resource.api.tasks import ExecutionRequest +from geonode.base.populate_test_data import ( + create_models, + create_single_dataset, + create_single_doc, + create_single_map, + create_single_geoapp, +) +from geonode.resource.api.tasks import resouce_service_dispatcher +from guardian.shortcuts import assign_perm + +logger = logging.getLogger(__name__) + +test_image = Image.new("RGBA", size=(50, 50), color=(155, 0, 0)) + + +class BaseApiTests(APITestCase): + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json", "test_thesaurus.json"] + + def setUp(self): + self.maxDiff = None + create_models(b"document") + create_models(b"map") + create_models(b"dataset") + + def test_groups_list(self): + """ + Ensure we can access the gropus list. + """ + pub_1 = GroupProfile.objects.create(slug="pub_1", title="pub_1", access="public") + priv_1 = GroupProfile.objects.create(slug="priv_1", title="priv_1", access="private") + priv_2 = GroupProfile.objects.create(slug="priv_2", title="priv_2", access="private") + pub_invite_1 = GroupProfile.objects.create(slug="pub_invite_1", title="pub_invite_1", access="public-invite") + pub_invite_2 = GroupProfile.objects.create(slug="pub_invite_2", title="pub_invite_2", access="public-invite") + try: + # Anonymous can access only public groups + url = reverse("group-profiles-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + logger.debug(response.data) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 4) + self.assertEqual(len(response.data["group_profiles"]), 4) + self.assertTrue(all([_g["access"] != "private" for _g in response.data["group_profiles"]])) + + # Admin can access all groups + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("group-profiles-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + logger.debug(response.data) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 6) + self.assertEqual(len(response.data["group_profiles"]), 6) + + # Bobby can access public groups and the ones he is member of + self.assertTrue(self.client.login(username="bobby", password="bob")) + priv_1.join(get_user_model().objects.get(username="bobby")) + url = reverse("group-profiles-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + logger.debug(response.data) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 5) + self.assertEqual(len(response.data["group_profiles"]), 5) + self.assertTrue(any([_g["slug"] == "priv_1" for _g in response.data["group_profiles"]])) + + url = reverse("group-profiles-detail", kwargs={"pk": priv_1.pk}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + logger.debug(response.data) + finally: + pub_1.delete() + priv_1.delete() + priv_2.delete() + pub_invite_1.delete() + pub_invite_2.delete() + + def test_create_group(self): + """ + Ensure only Admins can create groups. + """ + data = { + "title": "group title", + "group": 1, + "slug": "group_title", + "description": "test", + "access": "private", + "categories": [], + } + try: + # Anonymous + url = reverse("group-profiles-list") + response = self.client.post(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + + # Registered member + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.post(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + + # Group manager + group = GroupProfile.objects.create(slug="test_group_manager", title="test_group_manager") + group.join(get_user_model().objects.get(username="norman"), role="manager") + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.post(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.post(url, data=data, format="json") + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()["group_profile"]["title"], "group title") + finally: + GroupProfile.objects.get(slug="group_title").delete() + group.delete() + + def test_edit_group(self): + """ + Ensure only admins and group managers can edit a group. + """ + group = GroupProfile.objects.create(slug="pub_1", title="pub_1", access="public") + data = {"title": "new_title"} + try: + # Anonymous + url = f"{reverse('group-profiles-list')}/{group.id}/" + response = self.client.patch(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + + # Registered member + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.patch(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + + # Group manager + group.join(get_user_model().objects.get(username="bobby"), role="manager") + response = self.client.patch(url, data=data, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(GroupProfile.objects.get(id=group.id).title, data["title"]) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.patch(url, data={"title": "admin_title"}, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(GroupProfile.objects.get(id=group.id).title, "admin_title") + finally: + group.delete() + + def test_delete_group(self): + """ + Ensure only admins can delete a group. + """ + group = GroupProfile.objects.create(slug="pub_1", title="pub_1", access="public") + try: + # Anonymous + url = f"{reverse('group-profiles-list')}/{group.id}/" + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, 403) + + # Registered member + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, 403) + + # Group manager + group.join(get_user_model().objects.get(username="bobby"), role="manager") + response = self.client.delete(f"{reverse('group-profiles-list')}/{group.id}/", format="json") + self.assertEqual(response.status_code, 403) + + # Admin can delete a group + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.delete(f"{reverse('group-profiles-list')}/{group.id}/", format="json") + self.assertEqual(response.status_code, 204) + finally: + group.delete() + + def test_group_resources_shows_related_permissions(self): + """ + Calling the resources endpoint of the groups should return also the + group permission on that specific resource + """ + group = GroupProfile.objects.create(slug="group1", title="group1", access="public") + bobby = get_user_model().objects.filter(username="bobby").get() + GroupMember.objects.get_or_create(group=group, user=bobby, role="member") + dataset = Dataset.objects.first() + dataset.set_permissions({"groups": {group: ["base.view_resourcebase"]}}) + try: + self.assertTrue(self.client.login(username="bobby", password="bob")) + url = f"{reverse('group-profiles-list')}/{group.id}/resources" + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + perms = response.json().get("resources", [])[0].get("perms") + self.assertListEqual(["view_resourcebase"], perms) + finally: + group.delete() + + def test_users_list(self): + """ + Ensure we can access the users list. + """ + url = reverse("users-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 403) + # Authorized + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + logger.debug(response.data) + self.assertEqual(response.data["total"], 9) + self.assertEqual(len(response.data["users"]), 9) + # response has link to the response + self.assertTrue("link" in response.data["users"][0].keys()) + + url = reverse("users-detail", kwargs={"pk": 1}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + logger.debug(response.data) + self.assertEqual(response.data["user"]["username"], "admin") + self.assertIsNotNone(response.data["user"]["avatar"]) + + # anonymous users are not in contributors group + self.assertNotIn("add_resource", get_user_model().objects.get(id=-1).perms) + + try: + # Bobby + group_user = get_user_model().objects.create(username="group_user") + bobby = get_user_model().objects.filter(username="bobby").get() + groupx = GroupProfile.objects.create(slug="groupx", title="groupx", access="private") + groupx.join(bobby) + groupx.join(group_user) + self.assertTrue(self.client.login(username="bobby", password="bob")) + url = reverse("users-detail", kwargs={"pk": group_user.id}) + # Bobby can access other users details from same group + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + + # Bobby can see himself in the list + url = reverse("users-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertIn('"username": "bobby"', json.dumps(response.data["users"])) + + # Bobby can access its own details + url = reverse("users-detail", kwargs={"pk": bobby.id}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["user"]["username"], "bobby") + self.assertIsNotNone(response.data["user"]["avatar"]) + # default contributor group_perm is returned in perms + self.assertIn("add_resource", response.data["user"]["perms"]) + + # Bobby can't access other users perms list + url = reverse("users-detail", kwargs={"pk": group_user.id}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["user"]["username"], "group_user") + self.assertIsNotNone(response.data["user"]["avatar"]) + # default contributor group_perm is returned in perms + self.assertNotIn("perms", response.data["user"]) + finally: + group_user.delete() + groupx.delete() + + def test_user_resources_shows_related_permissions(self): + """ + Calling the resources endpoint of the user should return also the + user permission on that specific resource + """ + bobby = get_user_model().objects.filter(username="bobby").get() + dataset = Dataset.objects.first() + dataset.set_permissions({"users": {bobby: ["base.view_resourcebase", "base.change_resourcebase"]}}) + self.assertTrue(self.client.login(username="bobby", password="bob")) + url = f"{reverse('users-list')}/{bobby.id}/resources" + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + perms = response.json().get("resources", [])[0].get("perms") + self.assertSetEqual({"view_resourcebase", "change_resourcebase"}, set(perms)) + + def test_get_self_user_details_outside_registered_member(self): + try: + user = get_user_model().objects.create_user( + username="non_registered_member", email="non_registered_member@geonode.org", password="password" + ) + # remove user from registered members group + reg_mem_group = Group.objects.get(name="registered-members") + reg_mem_group.user_set.remove(user) + + url = reverse("users-detail", kwargs={"pk": user.pk}) + + self.assertTrue(self.client.login(username="non_registered_member", password="password")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + finally: + user.delete() + + def test_get_self_user_details_with_no_group(self): + try: + user = get_user_model().objects.create_user( + username="no_group_member", email="no_group_member@geonode.org", password="password" + ) + # remove user from all groups + user.groups.clear() + + url = reverse("users-detail", kwargs={"pk": user.pk}) + + self.assertTrue(self.client.login(username="no_group_member", password="password")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + finally: + user.delete() + + def test_register_users(self): + """ + Ensure users are created with default groups. + """ + url = reverse("users-list") + user_data = {"username": "new_user", "password": "@!2XJSL_S&V^0nt", "email": "user@exampl2e.com"} + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.post(url, data=user_data, format="json") + self.assertEqual(response.status_code, 201) + # default contributor group_perm is returned in perms + self.assertIn("add_resource", response.data["user"]["perms"]) + # Anonymous + self.assertIsNone(self.client.logout()) + response = self.client.post(url, data={"username": "new_user_1"}, format="json") + self.assertEqual(response.status_code, 403) + + def test_acess_profile_edit(self): + # Registered member + self.assertTrue(self.client.login(username="bobby", password="bob")) + user = get_user_model().objects.get(username="bobby") + + url = f'{reverse("profile_edit")}{user.username}' + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + + def test_update_user_profile(self): + """ + Ensure users cannot update others. + """ + try: + user = get_user_model().objects.create_user( + username="user_test_delete", email="user_test_delete@geonode.org", password="user" + ) + url = reverse("users-detail", kwargs={"pk": user.pk}) + data = {"first_name": "user", "password": "@!2XJSL_S&V^0nt", "email": "user@exampl2e.com"} + # Anonymous + response = self.client.patch(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + # Another registered user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.patch(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + # User self profile + self.assertTrue(self.client.login(username="user_test_delete", password="user")) + response = self.client.patch(url, data=data, format="json") + self.assertEqual(response.status_code, 200) + # Group manager + group = GroupProfile.objects.create(slug="test_group_manager", title="test_group_manager") + group.join(user) + group.join(get_user_model().objects.get(username="norman"), role="manager") + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.post(url, data=data, format="json") + # malformed url on post + self.assertEqual(response.status_code, 405) + # Admin can edit user + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.patch(url, data={"first_name": "user_admin"}, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(get_user_model().objects.get(username="user_test_delete").first_name, "user_admin") + finally: + user.delete() + group.delete() + + def test_delete_user_profile(self): + """ + Ensure only admins can delete profiles. + """ + try: + user = get_user_model().objects.create_user( + username="user_test_delete", email="user_test_delete@geonode.org", password="user" + ) + url = reverse("users-detail", kwargs={"pk": user.pk}) + # Anonymous can't read + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 403) + # Anonymous can't delete user + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, 403) + # Bob can't delete user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, 403) + # User can delete self profile + self.assertTrue(self.client.login(username="user_test_delete", password="user")) + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(get_user_model().objects.filter(username="user_test_delete").first(), None) + # recreate user that was deleted + user = get_user_model().objects.create_user( + username="user_test_delete", email="user_test_delete@geonode.org", password="user" + ) + url = reverse("users-detail", kwargs={"pk": user.pk}) + # Admin can delete user + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, 200) + finally: + user.delete() + + def test_base_resources(self): + """ + Ensure we can access the Resource Base list. + """ + url = reverse("base-resources-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 28) + + url = f"{reverse('base-resources-list')}?filter{{metadata_only}}=false" + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + response.data["resources"][0].get("executions") + # Pagination + self.assertEqual(len(response.data["resources"]), 10) + logger.debug(response.data) + + # Remove public permissions to Layers + from geonode.layers.utils import set_datasets_permissions + + set_datasets_permissions( + "read", # permissions_name + None, # resources_names == None (all layers) + [get_anonymous_user()], # users_usernames + None, # groups_names + True, # delete_flag + ) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + # Pagination + self.assertEqual(len(response.data["resources"]), 10) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + # response has link to the response + self.assertTrue("link" in response.data["resources"][0].keys()) + # Pagination + self.assertEqual(len(response.data["resources"]), 10) + logger.debug(response.data) + + # Bobby + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + # Pagination + self.assertEqual(len(response.data["resources"]), 10) + logger.debug(response.data) + + # Norman + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + # Pagination + self.assertEqual(len(response.data["resources"]), 10) + logger.debug(response.data) + + # Pagination + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get(f"{url}&page_size=17", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + # Pagination + self.assertEqual(len(response.data["resources"]), 17) + + # Check user permissions + resource = ResourceBase.objects.filter(owner__username="bobby").first() + self.assertEqual(resource.owner.username, "bobby") + # Admin + url_with_id = f"{reverse('base-resources-list')}/{resource.id}?filter{{metadata_only}}=false" + + response = self.client.get(f"{url_with_id}", format="json") + self.assertEqual(response.data["resource"]["state"], enumerations.STATE_PROCESSED) + self.assertEqual(response.data["resource"]["sourcetype"], enumerations.SOURCE_TYPE_LOCAL) + self.assertTrue("change_resourcebase" in list(response.data["resource"]["perms"])) + # Annonymous + self.assertIsNone(self.client.logout()) + response = self.client.get(f"{url_with_id}", format="json") + self.assertFalse("change_resourcebase" in list(response.data["resource"]["perms"])) + # user owner + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(f"{url_with_id}", format="json") + self.assertTrue("change_resourcebase" in list(response.data["resource"]["perms"])) + # user not owner and not assigned + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.get(f"{url_with_id}", format="json") + self.assertFalse("change_resourcebase" in list(response.data["resource"]["perms"])) + # Check executions are returned when deffered + # all resources + response = self.client.get(f"{url}&include[]=executions", format="json") + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(response.data["resources"][0].get("executions")) + # specific resource + exec_req = ExecutionRequest.objects.create( + user=resource.owner, + func_name="test", + geonode_resource=resource, + input_params={ + "uuid": resource.uuid, + "owner": resource.owner.username, + "resource_type": resource.resource_type, + "defaults": f'{{"owner":"{resource.owner.username}"}}', + }, + ) + expected_executions_results = [ + { + "exec_id": exec_req.exec_id, + "user": exec_req.user.username, + "status": exec_req.status, + "func_name": exec_req.func_name, + "created": exec_req.created, + "finished": exec_req.finished, + "last_updated": exec_req.last_updated, + "input_params": exec_req.input_params, + "output_params": exec_req.output_params, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": exec_req.exec_id}) + ), + } + ] + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(f"{url_with_id}&include[]=executions", format="json") + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(response.data["resource"].get("executions")) + self.assertEqual(response.data["resource"].get("executions"), expected_executions_results) + + # test 'tkeywords' + try: + for _tkw in ThesaurusKeyword.objects.filter(pk__gte=34): + resource.tkeywords.add(_tkw) + self.assertEqual(6, resource.tkeywords.count()) + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(f"{url_with_id}", format="json") + self.assertIsNotNone(response.data["resource"]["tkeywords"]) + self.assertEqual(6, len(response.data["resource"]["tkeywords"])) + self.assertListEqual( + [ + { + "name": "", + "slug": "http-inspire-ec-europa-eu-theme-37", + "uri": "http://inspire.ec.europa.eu/theme#37", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", + }, + "i18n": {}, + }, + { + "name": "", + "slug": "http-localhost-8000-thesaurus-no-about-thesauro-38", + "uri": "http://localhost:8000//thesaurus/no-about-thesauro#38", + "thesaurus": {"name": "Thesauro without the about", "slug": "no-about-thesauro", "uri": ""}, + "i18n": {}, + }, + { + "name": "bar_keyword", + "slug": "http-localhost-8000-thesaurus-no-about-thesauro-bar-keyword", + "uri": "http://localhost:8000//thesaurus/no-about-thesauro#bar_keyword", + "thesaurus": {"name": "Thesauro without the about", "slug": "no-about-thesauro", "uri": ""}, + "i18n": {}, + }, + { + "name": "foo_keyword", + "slug": "http-inspire-ec-europa-eu-theme-foo-keyword", + "uri": "http://inspire.ec.europa.eu/theme#foo_keyword", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", + }, + "i18n": {}, + }, + { + "name": "mf", + "slug": "http-inspire-ec-europa-eu-theme-mf", + "uri": "http://inspire.ec.europa.eu/theme/mf", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", + }, + "i18n": {"en": "Meteorological geographical features"}, + }, + { + "name": "us", + "slug": "http-inspire-ec-europa-eu-theme-us", + "uri": "http://inspire.ec.europa.eu/theme/us", + "thesaurus": { + "name": "GEMET - INSPIRE themes, version 1.0", + "slug": "inspire-theme", + "uri": "http://inspire.ec.europa.eu/theme", + }, + "i18n": {"en": "Utility and governmental services"}, + }, + ], + response.data["resource"]["tkeywords"], + ) + finally: + resource.tkeywords.set(ThesaurusKeyword.objects.none()) + self.assertEqual(0, resource.tkeywords.count()) + + def test_write_resources(self): + """ + Ensure we can perform write operation against the Resource Bases. + """ + url = reverse("base-resources-list") + # Check user permissions + for resource_type in ["dataset", "document", "map"]: + resource = ResourceBase.objects.filter(owner__username="bobby", resource_type=resource_type).first() + self.assertEqual(resource.owner.username, "bobby") + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(f"{url}/{resource.id}/", format="json") + self.assertTrue("change_resourcebase" in list(response.data["resource"]["perms"])) + self.assertEqual(True, response.data["resource"]["is_published"], response.data["resource"]["is_published"]) + data = { + "title": "Foo Title", + "abstract": "Foo Abstract", + "attribution": "Foo Attribution", + "doi": "321-12345-987654321", + "is_published": False, + } + response = self.client.patch(f"{url}/{resource.id}/", data=data, format="json") + self.assertEqual(response.status_code, 200, response.status_code) + response = self.client.get(f"{url}/{resource.id}/", format="json") + self.assertEqual("Foo Title", response.data["resource"]["title"], response.data["resource"]["title"]) + self.assertEqual( + "Foo Abstract", response.data["resource"]["abstract"], response.data["resource"]["abstract"] + ) + self.assertEqual( + "Foo Attribution", response.data["resource"]["attribution"], response.data["resource"]["attribution"] + ) + self.assertEqual("321-12345-987654321", response.data["resource"]["doi"], response.data["resource"]["doi"]) + self.assertEqual( + False, response.data["resource"]["is_published"], response.data["resource"]["is_published"] + ) + + def test_resource_serializer_validation(self): + """ + Testing serializing and deserializing of a Dataset base on django-rest description: + https://www.django-rest-framework.org/api-guide/serializers/#deserializing-objects + """ + owner, _ = get_user_model().objects.get_or_create(username="delet-owner") + title = "TEST DS TITLE" + HierarchicalKeyword.add_root(name="a") + keyword = HierarchicalKeyword.objects.get(slug="a") + keyword.add_child(name="a1") + + Dataset( + title=title, + abstract="abstract", + name="test dataset", + alternate="Test Remove User", + attribution="Test Attribution", + uuid=str(uuid4()), + doi="test DOI", + edition=1, + maintenance_frequency=enumerations.UPDATE_FREQUENCIES[0], + constraints_other="Test Constrains other", + temporal_extent_start=date.today() - timedelta(days=1), + temporal_extent_end=date.today(), + data_quality_statement="Test data quality statement", + purpose="Test Purpose", + owner=owner, + subtype="raster", + category=TopicCategory.objects.get(identifier="elevation"), + resource_type="dataset", + license=License.objects.all().first(), + restriction_code_type=RestrictionCodeType.objects.all().first(), + group=Group.objects.all().first(), + ).save() + + ds = ResourceBase.objects.get(title=title) + ds.keywords.add(HierarchicalKeyword.objects.get(slug="a1")) + + factory = RequestFactory() + rq = factory.get("test") + rq.user = owner + + serialized = ResourceBaseSerializer(ds, context={"request": rq}) + json = JSONRenderer().render(serialized.data) + stream = BytesIO(json) + data = JSONParser().parse(stream) + self.assertIsInstance(data, dict) + se = ResourceBaseSerializer(data=data, context={"request": rq}) + self.assertTrue(se.is_valid()) + + def test_resource_base_serializer_with_settingsfield(self): + doc = create_single_doc("my_custom_doc") + factory = RequestFactory() + rq = factory.get("test") + rq.user = doc.owner + serialized = ResourceBaseSerializer(doc, context={"request": rq}) + json = JSONRenderer().render(serialized.data) + stream = BytesIO(json) + data = JSONParser().parse(stream) + self.assertTrue(data.get("is_approved")) + self.assertTrue(data.get("is_published")) + self.assertFalse(data.get("featured")) + + def test_resource_settings_field(self): + """ + Admin is able to change the is_published value + """ + doc = create_single_doc("my_custom_doc") + factory = RequestFactory() + rq = factory.get("test") + rq.user = doc.owner + serializer = ResourceBaseSerializer(doc, context={"request": rq}) + field = serializer.fields["is_published"] + self.assertIsNotNone(field) + self.assertTrue(field.to_internal_value(True)) + + def test_resource_settings_field_non_admin(self): + """ + Non-Admin is not able to change the is_published value + if he is not the owner of the resource + """ + doc = create_single_doc("my_custom_doc") + factory = RequestFactory() + rq = factory.get("test") + rq.user = get_user_model().objects.get(username="bobby") + serializer = ResourceBaseSerializer(doc, context={"request": rq}) + field = serializer.fields["is_published"] + self.assertIsNotNone(field) + # the original value was true, so it should not return false + self.assertTrue(field.to_internal_value(False)) + + def test_delete_user_with_resource(self): + owner, created = get_user_model().objects.get_or_create(username="delet-owner") + Dataset( + title="Test Remove User", + abstract="abstract", + name="Test Remove User", + alternate="Test Remove User", + uuid=str(uuid4()), + owner=owner, + subtype="raster", + category=TopicCategory.objects.get(identifier="elevation"), + ).save() + # Delete user and check if default user is updated + owner.delete() + self.assertEqual( + ResourceBase.objects.get(title="Test Remove User").owner, get_user_model().objects.get(username="admin") + ) + + def test_search_resources(self): + """ + Ensure we can search across the Resource Base list. + """ + url = reverse("base-resources-list") + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get(f"{url}?search=ca&search_fields=title&search_fields=abstract", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 1) + # Pagination + self.assertEqual(len(response.data["resources"]), 1) + + def test_filter_resources(self): + """ + Ensure we can filter across the Resource Base list. + """ + url = reverse("base-resources-list") + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + # Filter by owner == bobby + response = self.client.get(f"{url}?filter{{owner.username}}=bobby&filter{{metadata_only}}=false", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 3) + # Pagination + self.assertEqual(len(response.data["resources"]), 3) + + # Filter by resource_type == document + response = self.client.get( + f"{url}?filter{{resource_type}}=document&filter{{metadata_only}}=false", format="json" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 9) + # Pagination + self.assertEqual(len(response.data["resources"]), 9) + + # Filter by resource_type == layer and title like 'common morx' + response = self.client.get( + f"{url}?filter{{resource_type}}=dataset&filter{{title.icontains}}=common morx&filter{{metadata_only}}=false", + format="json", + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 1) + # Pagination + self.assertEqual(len(response.data["resources"]), 1) + + # Filter by Keywords + response = self.client.get(f"{url}?filter{{keywords.name}}=here&filter{{metadata_only}}=false", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 1) + # Pagination + self.assertEqual(len(response.data["resources"]), 1) + + # Filter by Metadata Regions + response = self.client.get( + f"{url}?filter{{regions.name.icontains}}=Italy&filter{{metadata_only}}=false", format="json" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 0) + # Pagination + self.assertEqual(len(response.data["resources"]), 0) + + # Filter by Metadata Categories + response = self.client.get( + f"{url}?filter{{category.identifier}}=elevation&filter{{metadata_only}}=false", format="json" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 6) + # Pagination + self.assertEqual(len(response.data["resources"]), 6) + + # Extent Filter + response = self.client.get( + f"{url}?page_size=26&extent=-180,-90,180,90&filter{{metadata_only}}=false", format="json" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + # Pagination + self.assertEqual(len(response.data["resources"]), 26) + + response = self.client.get( + f"{url}?page_size=26&extent=0,0,100,100&filter{{metadata_only}}=false", format="json" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 26) + # Pagination + self.assertEqual(len(response.data["resources"]), 26) + + response = self.client.get( + f"{url}?page_size=26&extent=-10,-10,-1,-1&filter{{metadata_only}}=false", format="json" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 12) + # Pagination + self.assertEqual(len(response.data["resources"]), 12) + + # Extent Filter: Crossing Dateline + extent = "-180.0000,56.9689,-162.5977,70.7435,155.9180,56.9689,180.0000,70.7435" + response = self.client.get(f"{url}?page_size=26&extent={extent}&filter{{metadata_only}}=false", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 12) + # Pagination + self.assertEqual(len(response.data["resources"]), 12) + + def test_sort_resources(self): + """ + Ensure we can sort the Resource Base list. + """ + url = reverse("base-resources-list") + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get(f"{url}?sort[]=title", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 28) + # Pagination + self.assertEqual(len(response.data["resources"]), 10) + + resource_titles = [] + for _r in response.data["resources"]: + resource_titles.append(_r["title"]) + sorted_resource_titles = sorted(resource_titles.copy()) + self.assertEqual(resource_titles, sorted_resource_titles) + + response = self.client.get(f"{url}?sort[]=-title", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 28) + # Pagination + self.assertEqual(len(response.data["resources"]), 10) + + resource_titles = [] + for _r in response.data["resources"]: + resource_titles.append(_r["title"]) + + reversed_resource_titles = sorted(resource_titles.copy()) + self.assertNotEqual(resource_titles, reversed_resource_titles) + + def test_perms_resources(self): + """ + Ensure we can Get & Set Permissions across the Resource Base list. + """ + url = reverse("base-resources-list") + bobby = get_user_model().objects.get(username="bobby") + norman = get_user_model().objects.get(username="norman") + anonymous_group = Group.objects.get(name="anonymous") + contributors_group = Group.objects.get(name="registered-members") + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + resource = ResourceBase.objects.filter(owner__username="bobby").first() + set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") + get_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") + + url = reverse("base-resources-detail", kwargs={"pk": resource.pk}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(int(response.data["resource"]["pk"]), int(resource.pk)) + + response = self.client.get(get_perms_url, format="json") + self.assertEqual(response.status_code, 200) + resource_perm_spec = response.data + self.assertDictEqual( + resource_perm_spec, + { + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "owner", + "is_staff": False, + "is_superuser": False, + }, + { + "avatar": build_absolute_uri(avatar_url(bobby)), + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, + ], + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "download"}, + { + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", + }, + ], + }, + resource_perm_spec, + ) + + # Add perms to Norman + resource_perm_spec_patch = { + "uuid": resource.uuid, + "users": [ + { + "id": norman.id, + "username": norman.username, + "first_name": norman.first_name, + "last_name": norman.last_name, + "avatar": "", + "permissions": "edit", + "is_staff": False, + "is_superuser": False, + } + ], + } + response = self.client.patch(set_perms_url, data=resource_perm_spec_patch, format="json") + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(response.data.get("status")) + self.assertIsNotNone(response.data.get("status_url")) + status = response.data.get("status") + resp_js = json.loads(response.content.decode("utf-8")) + status_url = resp_js.get("status_url", None) + execution_id = resp_js.get("execution_id", "") + self.assertIsNotNone(status_url) + self.assertIsNotNone(execution_id) + for _cnt in range(0, 10): + response = self.client.get(f"{status_url}") + self.assertEqual(response.status_code, 200) + resp_js = json.loads(response.content.decode("utf-8")) + logger.error(f"[{_cnt + 1}] ... {resp_js}") + if resp_js.get("status", "") == "finished": + status = resp_js.get("status", "") + break + else: + resouce_service_dispatcher.apply((execution_id,)) + sleep(3.0) + self.assertTrue(status, ExecutionRequest.STATUS_FINISHED) + + response = self.client.get(get_perms_url, format="json") + self.assertEqual(response.status_code, 200) + resource_perm_spec = response.data + self.assertDictEqual( + resource_perm_spec, + { + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "owner", + "is_staff": False, + "is_superuser": False, + }, + { + "id": norman.id, + "username": norman.username, + "first_name": norman.first_name, + "last_name": norman.last_name, + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "edit", + "is_staff": False, + "is_superuser": False, + }, + { + "avatar": build_absolute_uri(avatar_url(bobby)), + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, + ], + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "download"}, + { + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", + }, + ], + }, + resource_perm_spec, + ) + + # Remove perms to Norman + resource_perm_spec = { + "uuid": resource.uuid, + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "owner", + "is_staff": False, + "is_superuser": False, + }, + { + "avatar": build_absolute_uri(avatar_url(bobby)), + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, + ], + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + { + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", + }, + ], + } + + response = self.client.put(set_perms_url, data=resource_perm_spec, format="json") + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(response.data.get("status")) + self.assertIsNotNone(response.data.get("status_url")) + status = response.data.get("status") + resp_js = json.loads(response.content.decode("utf-8")) + status_url = resp_js.get("status_url", None) + execution_id = resp_js.get("execution_id", "") + self.assertIsNotNone(status_url) + self.assertIsNotNone(execution_id) + for _cnt in range(0, 10): + response = self.client.get(f"{status_url}") + self.assertEqual(response.status_code, 200) + resp_js = json.loads(response.content.decode("utf-8")) + logger.error(f"[{_cnt + 1}] ... {resp_js}") + if resp_js.get("status", "") == "finished": + status = resp_js.get("status", "") + break + else: + resouce_service_dispatcher.apply((execution_id,)) + sleep(3.0) + self.assertTrue(status, ExecutionRequest.STATUS_FINISHED) + + response = self.client.get(get_perms_url, format="json") + self.assertEqual(response.status_code, 200) + resource_perm_spec = response.data + self.assertDictEqual( + resource_perm_spec, + { + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "owner", + "is_staff": False, + "is_superuser": False, + }, + { + "avatar": build_absolute_uri(avatar_url(bobby)), + "first_name": "admin", + "id": 1, + "last_name": "", + "permissions": "manage", + "username": "admin", + "is_staff": True, + "is_superuser": True, + }, + ], + "organizations": [], + "groups": [ + {"id": anonymous_group.id, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + { + "id": contributors_group.id, + "title": "Registered Members", + "name": contributors_group.name, + "permissions": "none", + }, + ], + }, + resource_perm_spec, + ) + + # Ensure get_perms and set_perms are done by users with correct permissions. + # logout admin user + self.assertIsNone(self.client.logout()) + # get perms + response = self.client.get(get_perms_url, format="json") + self.assertEqual(response.status_code, 403) + # set perms + resource_perm_spec["uuid"] = resource.uuid + response = self.client.put(set_perms_url, data=resource_perm_spec, format="json") + self.assertEqual(response.status_code, 403) + # login resourse owner + # get perms + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(get_perms_url, format="json") + self.assertEqual(response.status_code, 200) + # set perms + response = self.client.put(set_perms_url, data=resource_perm_spec, format="json") + self.assertEqual(response.status_code, 200) + + def test_featured_and_published_resources(self): + """ + Ensure we can Get & Set Permissions across the Resource Base list. + """ + url = reverse("base-resources-list") + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + resources = ResourceBase.objects.filter(owner__username="bobby", metadata_only=False) + + url = urljoin(f"{reverse('base-resources-list')}/", "featured/") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 0) + # Pagination + self.assertEqual(len(response.data["resources"]), 0) + + resources.filter(resource_type="map").update(featured=True) + url = urljoin(f"{reverse('base-resources-list')}/", "featured/") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], resources.filter(resource_type="map").count()) + # Pagination + self.assertEqual(len(response.data["resources"]), resources.filter(resource_type="map").count()) + + def test_resource_types(self): + """ + Ensure we can Get & Set Permissions across the Resource Base list. + """ + url = urljoin(f"{reverse('base-resources-list')}/", "resource_types/") + response = self.client.get(url, format="json") + r_types = [item for item in response.data["resource_types"]] + r_type_names = [r_type["name"] for r_type in r_types] + self.assertEqual(response.status_code, 200) + self.assertTrue("resource_types" in response.data) + self.assertTrue("dataset" in r_type_names) + self.assertTrue("map" in r_type_names) + self.assertTrue("document" in r_type_names) + self.assertFalse("service" in r_type_names) + + r_type_perms = {r_type["name"]: r_type["allowed_perms"] for r_type in r_types} + _pp_e = { + "perms": { + "anonymous": ["view_resourcebase", "download_resourcebase"], + "default": [ + "delete_resourcebase", + "view_resourcebase", + "change_resourcebase_metadata", + "change_resourcebase_permissions", + "publish_resourcebase", + "change_resourcebase", + "change_resourcebase_metadata", + "download_resourcebase", + "change_dataset_data", + "change_dataset_style", + ], + "registered-members": [ + "delete_resourcebase", + "view_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", + "change_resourcebase", + "change_resourcebase_metadata", + "download_resourcebase", + "change_dataset_data", + "change_dataset_style", + ], + }, + "compact": { + "anonymous": [ + {"name": "none", "label": "None"}, + {"name": "view", "label": "View"}, + {"name": "download", "label": "Download"}, + ], + "default": [ + {"name": "view", "label": "View"}, + {"name": "download", "label": "Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + {"name": "owner", "label": "Owner"}, + {"name": "owner", "label": "Owner"}, + ], + "registered-members": [ + {"name": "none", "label": "None"}, + {"name": "view", "label": "View"}, + {"name": "download", "label": "Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + ], + }, + } + self.assertListEqual( + list(r_type_perms["dataset"].keys()), + list(_pp_e.keys()), + f"dataset : {list(r_type_perms['dataset'].keys())}", + ) + for _key in r_type_perms["dataset"]["perms"].keys(): + self.assertListEqual( + sorted(list(set(r_type_perms["dataset"]["perms"].get(_key)))), + sorted(list(set(_pp_e["perms"].get(_key)))), + f"{_key} : {list(set(r_type_perms['dataset']['perms'].get(_key)))}", + ) + for _key in r_type_perms["dataset"]["compact"].keys(): + self.assertListEqual( + r_type_perms["dataset"]["compact"].get(_key), + _pp_e["compact"].get(_key), + f"{_key} : {r_type_perms['dataset']['compact'].get(_key)}", + ) + + _pp_e = { + "perms": { + "anonymous": ["view_resourcebase", "download_resourcebase"], + "default": [ + "change_resourcebase_metadata", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", + "change_resourcebase", + "view_resourcebase", + "download_resourcebase", + ], + "registered-members": [ + "change_resourcebase_metadata", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", + "change_resourcebase", + "view_resourcebase", + "download_resourcebase", + ], + }, + "compact": { + "anonymous": [ + {"name": "none", "label": "None"}, + {"name": "view", "label": "View Metadata"}, + {"name": "download", "label": "View and Download"}, + ], + "default": [ + {"name": "view", "label": "View Metadata"}, + {"name": "download", "label": "View and Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + {"name": "owner", "label": "Owner"}, + {"name": "owner", "label": "Owner"}, + ], + "registered-members": [ + {"name": "none", "label": "None"}, + {"name": "view", "label": "View Metadata"}, + {"name": "download", "label": "View and Download"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + ], + }, + } + self.assertListEqual( + list(r_type_perms["document"].keys()), + list(_pp_e.keys()), + f"document : {list(r_type_perms['document'].keys())}", + ) + for _key in r_type_perms["document"]["perms"].keys(): + self.assertListEqual( + sorted(list(set(r_type_perms["document"]["perms"].get(_key)))), + sorted(list(set(_pp_e["perms"].get(_key)))), + f"{_key} : {list(set(r_type_perms['document']['perms'].get(_key)))}", + ) + for _key in r_type_perms["document"]["compact"].keys(): + self.assertListEqual( + r_type_perms["document"]["compact"].get(_key), + _pp_e["compact"].get(_key), + f"{_key} : {r_type_perms['document']['compact'].get(_key)}", + ) + + _pp_e = { + "perms": { + "anonymous": [ + "view_resourcebase", + ], + "default": [ + "change_resourcebase_metadata", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", + "change_resourcebase", + "view_resourcebase", + ], + "registered-members": [ + "change_resourcebase_metadata", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", + "change_resourcebase", + "view_resourcebase", + ], + }, + "compact": { + "anonymous": [{"name": "none", "label": "None"}, {"name": "view", "label": "View"}], + "default": [ + {"name": "view", "label": "View"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + {"name": "owner", "label": "Owner"}, + {"name": "owner", "label": "Owner"}, + ], + "registered-members": [ + {"name": "none", "label": "None"}, + {"name": "view", "label": "View"}, + {"name": "edit", "label": "Edit"}, + {"name": "manage", "label": "Manage"}, + ], + }, + } + self.assertListEqual( + list(r_type_perms["map"].keys()), list(_pp_e.keys()), f"map : {list(r_type_perms['map'].keys())}" + ) + for _key in r_type_perms["map"]["perms"].keys(): + self.assertListEqual( + sorted(list(set(r_type_perms["map"]["perms"].get(_key)))), + sorted(list(set(_pp_e["perms"].get(_key)))), + f"{_key} : {list(set(r_type_perms['map']['perms'].get(_key)))}", + ) + for _key in r_type_perms["map"]["compact"].keys(): + self.assertListEqual( + r_type_perms["map"]["compact"].get(_key), + _pp_e["compact"].get(_key), + f"{_key} : {r_type_perms['map']['compact'].get(_key)}", + ) + + def test_get_favorites(self): + """ + Ensure we get user's favorite resources. + """ + dataset = Dataset.objects.first() + url = urljoin(f"{reverse('base-resources-list')}/", "favorites/") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 403) + # Authenticated user + bobby = get_user_model().objects.get(username="bobby") + self.assertTrue(self.client.login(username="bobby", password="bob")) + favorite = Favorite.objects.create_favorite(dataset, bobby) + response = self.client.get(url, format="json") + self.assertEqual(response.data["total"], 1) + self.assertEqual(response.status_code, 200) + # clean up + favorite.delete() + + def test_get_favorites_is_returned_in_the_base_endpoint_per_user(self): + """ + Ensure we get user's favorite resources. + """ + dataset = Dataset.objects.order_by("-last_updated").first() + url = reverse("base-resources-list") + bobby = get_user_model().objects.get(username="bobby") + + self.client.login(username="bobby", password="bob") + + favorite = Favorite.objects.create_favorite(dataset, bobby) + + response = self.client.get(url, format="json") + + self.assertEqual(response.status_code, 200) + resource_have_tag = [r.get("favorite", False) for r in response.json().get("resources", {})] + + # check that there is at last 1 favorite for the user + self.assertTrue(any(resource_have_tag)) + # clean up + favorite.delete() + + self.client.login(username="admin", password="admin") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + resource_have_tag = [r.get("favorite", False) for r in response.json().get("resources", {})] + # the admin should not have any favorite assigned to him + self.assertFalse(all(resource_have_tag)) + + def test_get_favorites_is_returned_in_the_base_endpoint(self): + """ + Ensure we get user's favorite resources. + """ + url = reverse("base-resources-list") + + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") + + self.assertEqual(response.status_code, 200) + resource_have_tag = ["favorite" in r.keys() for r in response.json().get("resources", {})] + self.assertTrue(all(resource_have_tag)) + + def test_create_and_delete_favorites(self): + """ + Ensure we can add and remove resources to user's favorite. + """ + bobby = get_user_model().objects.get(username="bobby") + dataset = create_single_dataset(name="test_dataset_for_fav", owner=bobby) + dataset.set_permissions({"users": {"bobby": ["base.add_resourcebase"]}}) + url = urljoin(f"{reverse('base-resources-list')}/", f"{dataset.pk}/favorite/") + # Anonymous + response = self.client.post(url, format="json") + self.assertEqual(response.status_code, 403) + # Authenticated user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.post(url, format="json") + self.assertEqual(response.data["message"], "Successfuly added resource to favorites") + self.assertEqual(response.status_code, 201) + # add resource to favorite again + response = self.client.post(url, format="json") + self.assertEqual(response.data["message"], "Resource is already in favorites") + self.assertEqual(response.status_code, 400) + # remove resource from favorites + response = self.client.delete(url, format="json") + self.assertEqual(response.data["message"], "Successfuly removed resource from favorites") + self.assertEqual(response.status_code, 200) + # remove resource to favorite again + response = self.client.delete(url, format="json") + self.assertEqual(response.data["message"], "Resource not in favorites") + self.assertEqual(response.status_code, 404) + dataset.delete() + + def test_search_resources_with_favorite_true_and_no_favorite_should_return_0(self): + """ + Ensure we can search across the Resource Base list. + """ + url = reverse("base-resources-list") + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get(f"{url}?favorite=true", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + # No favorite are saved, so the total should be 0 + self.assertEqual(response.data["total"], 0) + self.assertEqual(len(response.data["resources"]), 0) + + def test_search_resources_with_favorite_true_and_favorite_should_return_1(self): + """ + Ensure we can search across the Resource Base list. + """ + url = reverse("base-resources-list") + # Admin + admin = get_user_model().objects.get(username="admin") + dataset = Dataset.objects.first() + Favorite.objects.create_favorite(dataset, admin) + + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get(f"{url}?favorite=true", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + # 1 favorite is saved, so the total should be 1 + self.assertEqual(response.data["total"], 1) + self.assertEqual(len(response.data["resources"]), 1) + + def test_search_resources_with_favorite_true_with_geoapps_icluded(self): + url = reverse("base-resources-list") + # Admin + admin = get_user_model().objects.get(username="admin") + try: + geo_app = GeoApp.objects.create(title="Test GeoApp Favorite", owner=admin, resource_type="geostory") + Favorite.objects.create_favorite(geo_app, admin) + + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get(f"{url}?favorite=true", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 1) + self.assertEqual(len(response.data["resources"]), 1) + finally: + geo_app.delete() + + def test_thumbnail_urls(self): + """ + Ensure the thumbnail url reflects the current active Thumb on the resource. + """ + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + resource = ResourceBase.objects.filter(owner__username="bobby").first() + url = reverse("base-resources-detail", kwargs={"pk": resource.pk}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(int(response.data["resource"]["pk"]), int(resource.pk)) + thumbnail_url = response.data["resource"]["thumbnail_url"] + self.assertIsNone(thumbnail_url) + + def test_embed_urls(self): + """ + Ensure the embed urls reflect the concrete instance ones. + """ + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + resources = ResourceBase.objects.all() + for resource in resources: + url = f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}?filter{{metadata_only}}=false" + response = self.client.get(url, format="json") + if resource.title.endswith("metadata true"): + self.assertEqual(response.status_code, 404) + else: + self.assertEqual(response.status_code, 200) + self.assertEqual(int(response.data["resource"]["pk"]), int(resource.pk)) + embed_url = response.data["resource"]["embed_url"] + self.assertIsNotNone(embed_url) + + instance = resource.get_real_instance() + if hasattr(instance, "embed_url"): + if instance.embed_url != NotImplemented: + self.assertEqual(build_absolute_uri(instance.embed_url), embed_url) + else: + self.assertEqual("", embed_url) + + def test_owners_list(self): + """ + Ensure we can access the list of owners. + """ + url = reverse("owners-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 8) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 8) + response = self.client.get(f"{url}?type=geoapp", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 0) + response = self.client.get(f"{url}?type=dataset&title__icontains=CA", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 1) + # response has link to the response + self.assertTrue("link" in response.data["owners"][0].keys()) + + # Authenticated user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 8) + + # Owners Filtering + response = self.client.get(f"{url}?filter{{username.icontains}}=bobby", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 1) + + def test_categories_list(self): + """ + Ensure we can access the list of categories. + """ + url = reverse("categories-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], TopicCategory.objects.count()) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], TopicCategory.objects.count()) + # response has link to the response + self.assertTrue("link" in response.data["categories"][0].keys()) + + # Authenticated user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], TopicCategory.objects.count()) + + # Categories Filtering + response = self.client.get(f"{url}?filter{{identifier.icontains}}=biota", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 1) + + def test_regions_list(self): + """ + Ensure we can access the list of regions. + """ + url = reverse("regions-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], Region.objects.count()) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], Region.objects.count()) + # response has link to the response + self.assertTrue("link" in response.data["regions"][0].keys()) + + # Authenticated user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], Region.objects.count()) + + # Regions Filtering + response = self.client.get(f"{url}?filter{{name.icontains}}=Africa", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 8) + + def test_regions_with_resources(self): + """ + Ensure we can access the list of regions. + """ + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("regions-list") + response = self.client.get(f"{url}?with_resources=True", format="json") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 0) + + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("regions-list") + response = self.client.get(f"{url}?with_resources=False", format="json") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], Region.objects.count()) + + self.assertTrue(self.client.login(username="admin", password="admin")) + url = reverse("regions-list") + response = self.client.get(f"{url}", format="json") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], Region.objects.count()) + + def test_keywords_list(self): + """ + Ensure we can access the list of keywords. + """ + url = reverse("keywords-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], len(HierarchicalKeyword.resource_keywords_tree(None))) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + admin = get_user_model().objects.get(username="admin") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], len(HierarchicalKeyword.resource_keywords_tree(admin))) + # response has link to the response + if response.data["total"] > 0: + self.assertTrue("link" in response.data["keywords"][0].keys()) + + # Authenticated user + self.assertTrue(self.client.login(username="bobby", password="bob")) + bobby = get_user_model().objects.get(username="bobby") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], len(HierarchicalKeyword.resource_keywords_tree(bobby))) + + # Keywords Filtering + response = self.client.get(f"{url}?filter{{name.icontains}}=Africa", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], 0) + + # Testing Hierarchical Keywords tree + try: + HierarchicalKeyword.objects.filter(slug__in=["a", "a1", "a2", "b", "b1"]).delete() + HierarchicalKeyword.add_root(name="a") + HierarchicalKeyword.add_root(name="b") + a = HierarchicalKeyword.objects.get(slug="a") + b = HierarchicalKeyword.objects.get(slug="b") + a.add_child(name="a1") + a.add_child(name="a2") + b.add_child(name="b1") + resources = ResourceBase.objects.filter(owner__username="bobby") + res1 = resources.first() + res2 = resources.last() + self.assertNotEqual(res1, res2) + res1.keywords.add(HierarchicalKeyword.objects.get(slug="a1")) + res1.keywords.add(HierarchicalKeyword.objects.get(slug="a2")) + res2.keywords.add(HierarchicalKeyword.objects.get(slug="b1")) + resource_keywords = HierarchicalKeyword.resource_keywords_tree(bobby) + logger.error(resource_keywords) + + """ + Final api outcome will be something like + [ + { + 'id': 116, + 'text': 'a', + 'href': 'a', + 'tags': [], + 'nodes': [ + { + 'id': 118, + 'text': 'a1', + 'href': 'a1', + 'tags': [1] + }, + { + 'id': 119, + 'text': 'a2', + 'href': 'a2', + 'tags': [1] + } + ] + }, + { + 'id': 117, + 'text': 'b', + 'href': 'b', + 'tags': [], + 'nodes': [ + { + 'id': 120, + 'text': 'b1', + 'href': 'b1', + 'tags': [1] + } + ] + }, + ... + ] + """ + url = reverse("keywords-list") + # Authenticated user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + for _kw in response.data["keywords"]: + if _kw.get("href") in ["a", "b"]: + self.assertListEqual(_kw.get("tags"), [], _kw.get("tags")) + self.assertEqual(len(_kw.get("nodes")), 2) + for _kw_child in _kw.get("nodes"): + self.assertEqual(_kw_child.get("tags")[0], 1) + finally: + HierarchicalKeyword.objects.filter(slug__in=["a", "a1", "a2", "b", "b1"]).delete() + + def test_tkeywords_list(self): + """ + Ensure we can access the list of thasaurus keywords. + """ + url = reverse("tkeywords-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], ThesaurusKeyword.objects.count()) + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], ThesaurusKeyword.objects.count()) + # response has uri to the response + self.assertTrue("uri" in response.data["tkeywords"][0].keys()) + + # Authenticated user + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["total"], ThesaurusKeyword.objects.count()) + + def test_set_resource_thumbnail(self): + re_uuid = "[0-F]{8}-([0-F]{4}-){3}[0-F]{12}" + resource = Dataset.objects.first() + url = reverse("base-resources-set_thumbnail", args=[resource.pk]) + data = {"file": "http://localhost:8000/thumb.png"} + + # Anonymous user + response = self.client.put(url, data=data, format="json") + self.assertEqual(response.status_code, 403) + + # Authenticated user + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.put(url, data=data, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["thumbnail_url"], data["file"]) + self.assertEqual(Dataset.objects.get(pk=resource.pk).thumbnail_url, data["file"]) + # set with invalid image url + data = {"file": "invali url"} + response = self.client.put(url, data=data, format="json") + self.assertEqual(response.status_code, 400) + expected = { + "success": False, + "errors": ["file is either a file upload, ASCII byte string or a valid image url string"], + "code": "invalid", + } + self.assertEqual(response.json(), expected) + # Test with non image url + data = {"file": "http://localhost:8000/thumb.txt"} + response = self.client.put(url, data=data, format="json") + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), "The url must be of an image with format (png, jpeg or jpg)") + + # using Base64 data as an ASCII byte string + data["file"] = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABHNCSVQICAgI\ + fAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAAANSURBVAiZYzAxMfkPAALYAZzx61+bAAAAAElFTkSuQmCC" + ) + with patch("geonode.base.models.is_monochromatic_image") as _mck: + _mck.return_value = False + response = self.client.put(url, data=data, format="json") + self.assertEqual(response.status_code, 200) + self.assertIsNotNone( + re.search( + f"dataset-{re_uuid}-thumb-{re_uuid}.jpg", Dataset.objects.get(pk=resource.pk).thumbnail_url, re.I + ) + ) + # File upload + with patch("PIL.Image.open") as _mck: + _mck.return_value = test_image + # rest thumbnail_url to None + resource.thumbnail_url = None + resource.save() + self.assertEqual(Dataset.objects.get(pk=resource.pk).thumbnail_url, None) + f = SimpleUploadedFile("test_image.png", BytesIO(test_image.tobytes()).read(), "image/png") + response = self.client.put(url, data={"file": f}) + self.assertIsNotNone( + re.search( + f"dataset-{re_uuid}-thumb-{re_uuid}.jpg", + Dataset.objects.get(pk=resource.pk).thumbnail_url, + re.I, + ) + ) + self.assertEqual(response.status_code, 200) + + def test_set_thumbnail_from_bbox_from_Anonymous_user_raise_permission_error(self): + """ + Given a request with Anonymous user, should raise an authentication error. + """ + dataset_id = sys.maxsize + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) + # Anonymous + expected = { + "success": False, + "errors": ["Authentication credentials were not provided."], + "code": "not_authenticated", + } + response = self.client.post(url, format="json") + self.assertEqual(response.status_code, 403) + self.assertEqual(expected, response.json()) + + @patch("geonode.base.api.views.create_thumbnail") + def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset(self, mock_create_thumbnail): + """ + Given a logged User and an existing dataset, should create the expected thumbnail url. + """ + mock_create_thumbnail.return_value = "http://localhost:8000/mocked_url.jpg" + # Admin + self.client.login(username="admin", password="admin") + dataset_id = Dataset.objects.first().resourcebase_ptr_id + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) + payload = { + "bbox": [-9072629.904175375, -9043966.018568434, 1491839.8773032012, 1507127.2829602365], + "srid": "EPSG:3857", + } + response = self.client.post(url, data=payload, format="json") + + expected = { + "message": "Thumbnail correctly created.", + "success": True, + "thumbnail_url": "http://localhost:8000/mocked_url.jpg", + } + self.assertEqual(response.status_code, 200) + self.assertEqual(expected, response.json()) + + def test_set_thumbnail_from_bbox_from_logged_user_for_not_existing_dataset(self): + """ + Given a logged User and an not existing dataset, should raise a 404 error. + """ + # Admin + self.client.login(username="admin", password="admin") + dataset_id = sys.maxsize + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) + payload = { + "bbox": [-9072629.904175375, -9043966.018568434, 1491839.8773032012, 1507127.2829602365], + "srid": "EPSG:3857", + } + response = self.client.post(url, data=payload, format="json") + + expected = {"message": f"Resource selected with id {dataset_id} does not exists", "success": False} + self.assertEqual(response.status_code, 404) + self.assertEqual(expected, response.json()) + + def test_set_thumbnail_from_bbox_from_logged_user_for_existing_doc(self): + """ + Given a logged User and an existing doc, should raise a NotImplemented. + """ + # Admin + self.client.login(username="admin", password="admin") + dataset_id = Document.objects.first().resourcebase_ptr_id + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) + payload = {"bbox": [], "srid": "EPSG:3857"} + response = self.client.post(url, data=payload, format="json") + + expected = {"message": "Not implemented: Endpoint available only for Dataset and Maps", "success": False} + self.assertEqual(response.status_code, 405) + self.assertEqual(expected, response.json()) + + @patch( + "geonode.base.api.views.create_thumbnail", side_effect=ThumbnailError("Some exception during thumb creation") + ) + def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset_raise_exp(self, mock_exp): + """ + Given a logged User and an existing dataset, should raise a ThumbnailException. + """ + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + dataset_id = Dataset.objects.first().resourcebase_ptr_id + url = reverse("base-resources-set-thumb-from-bbox", args=[dataset_id]) + payload = {"bbox": [], "srid": "EPSG:3857"} + response = self.client.post(url, data=payload, format="json") + + expected = {"message": "Some exception during thumb creation", "success": False} + self.assertEqual(response.status_code, 500) + self.assertEqual(expected, response.json()) + + def test_manager_can_edit_map(self): + """ + REST API must not forbid saving maps and apps to non-admin and non-owners. + """ + self.maxDiff = None + from geonode.maps.models import Map + + _map = Map.objects.filter(uuid__isnull=False, owner__username="admin").first() + if not len(_map.uuid): + _map.uuid = str(uuid4()) + _map.save() + resource = ResourceBase.objects.filter(uuid=_map.uuid).first() + bobby = get_user_model().objects.get(username="bobby") + + # Add perms to Bobby + resource_perm_spec_patch = { + "uuid": resource.uuid, + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "", + "permissions": "manage", + } + ], + } + + # Patch the resource perms + self.assertTrue(self.client.login(username="admin", password="admin")) + set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") + response = self.client.patch(set_perms_url, data=resource_perm_spec_patch, format="json") + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(response.data.get("status")) + self.assertIsNotNone(response.data.get("status_url")) + status = response.data.get("status") + resp_js = json.loads(response.content.decode("utf-8")) + status_url = resp_js.get("status_url", None) + execution_id = resp_js.get("execution_id", "") + self.assertIsNotNone(status_url) + self.assertIsNotNone(execution_id) + for _cnt in range(0, 10): + response = self.client.get(f"{status_url}") + self.assertEqual(response.status_code, 200) + resp_js = json.loads(response.content.decode("utf-8")) + logger.error(f"[{_cnt + 1}] ... {resp_js}") + if resp_js.get("status", "") == "finished": + status = resp_js.get("status", "") + break + else: + resouce_service_dispatcher.apply((execution_id,)) + sleep(3.0) + self.assertTrue(status, ExecutionRequest.STATUS_FINISHED) + + # Test "bobby" can access the "permissions" endpoint + resource_service_permissions_url = reverse("base-resources-perms-spec", kwargs={"pk": resource.pk}) + response = self.client.get(resource_service_permissions_url, format="json") + self.assertEqual(response.status_code, 200) + resource_perm_spec = response.data + self.assertDictEqual( + resource_perm_spec, + { + "users": [ + { + "id": bobby.id, + "username": "bobby", + "first_name": "bobby", + "last_name": "", + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "manage", + "is_superuser": False, + "is_staff": False, + }, + { + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "owner", + "is_superuser": True, + "is_staff": True, + }, + ], + "organizations": [], + "groups": [ + {"id": 3, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + {"id": 2, "title": "Registered Members", "name": "registered-members", "permissions": "none"}, + ], + }, + resource_perm_spec, + ) + + # Test "bobby" can manage the resource permissions + get_perms_url = urljoin( + f"{reverse('base-resources-detail', kwargs={'pk': _map.get_self_resource().pk})}/", "permissions" + ) + response = self.client.get(get_perms_url, format="json") + self.assertEqual(response.status_code, 200) + resource_perm_spec = response.data + self.assertDictEqual( + resource_perm_spec, + { + "users": [ + { + "id": bobby.id, + "username": "bobby", + "first_name": "bobby", + "last_name": "", + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "manage", + "is_staff": False, + "is_superuser": False, + }, + { + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "owner", + "is_staff": True, + "is_superuser": True, + }, + ], + "organizations": [], + "groups": [ + {"id": 3, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + {"id": 2, "title": "Registered Members", "name": "registered-members", "permissions": "none"}, + ], + }, + resource_perm_spec, + ) + + # Fetch the map perms as user "bobby" + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.get(get_perms_url, format="json") + self.assertEqual(response.status_code, 200) + resource_perm_spec = response.data + self.assertDictEqual( + resource_perm_spec, + { + "users": [ + { + "id": bobby.id, + "username": "bobby", + "first_name": "bobby", + "last_name": "", + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "manage", + "is_staff": False, + "is_superuser": False, + }, + { + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": build_absolute_uri(avatar_url(bobby)), + "permissions": "owner", + "is_staff": True, + "is_superuser": True, + }, + ], + "organizations": [], + "groups": [ + {"id": 3, "title": "anonymous", "name": "anonymous", "permissions": "view"}, + {"id": 2, "title": "Registered Members", "name": "registered-members", "permissions": "none"}, + ], + }, + resource_perm_spec, + ) + + def test_resource_service_copy(self): + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") + files_as_dict, _ = get_files(files) + resource = resource_manager.create( + str(uuid4()), + Dataset, + defaults={ + "owner": get_user_model().objects.get(username="admin"), + "name": "test_copy", + "store": "geonode_data", + "subtype": "vector", + "alternate": "geonode:test_copy", + }, + ) + + asset, link = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) + bobby = get_user_model().objects.get(username="bobby") + copy_url = reverse("importer_resource_copy", kwargs={"pk": resource.pk}) + response = self.client.put(copy_url, data={"title": "cloned_resource"}) + self.assertEqual(response.status_code, 403) + # set perms to enable user clone resource + self.assertTrue(self.client.login(username="admin", password="admin")) + perm_spec = { + "uuid": resource.uuid, + "users": [ + { + "id": bobby.id, + "username": bobby.username, + "first_name": bobby.first_name, + "last_name": bobby.last_name, + "avatar": "", + "permissions": "manage", + } + ], + } + set_perms_url = urljoin(f"{reverse('base-resources-detail', kwargs={'pk': resource.pk})}/", "permissions") + + response = self.client.patch(set_perms_url, data=perm_spec, format="json") + self.assertEqual(response.status_code, 200) + # clone resource + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.put(copy_url) + self.assertEqual(response.status_code, 200) + cloned_resource = Dataset.objects.last() + self.assertEqual(cloned_resource.owner.username, "admin") + + # Check that the 'featured' flag is reset on clone + resource.refresh_from_db() + resource.featured = True + resource.save() + response = self.client.put(copy_url) + self.assertEqual(response.status_code, 200) + second_cloned = Dataset.objects.exclude(pk__in=[resource.pk, cloned_resource.pk]).latest("id") + self.assertFalse(second_cloned.featured, msg="Cloned resource should have featured=False") + + # clone dataset with invalid file + # resource.files = ["/path/invalid_file.wrong"] + # resource.save() + asset.location = ["/path/invalid_file.wrong"] + asset.save() + response = self.client.put(copy_url) + + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()["message"], "Resource can not be cloned.") + # clone dataset with no files + link.delete() + asset.delete() + response = self.client.put(copy_url) + + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()["message"], "Resource can not be cloned.") + # clean + try: + resource.delete() + except Exception as e: + logger.warning(f"Can't delete test resource {resource}", exc_info=e) + + def test_resource_service_copy_with_perms_dataset(self): + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") + files_as_dict, _ = get_files(files) + resource = Dataset.objects.create( + owner=get_user_model().objects.get(username="admin"), + name="test_copy", + store="geonode_data", + subtype="vector", + alternate="geonode:test_copy", + resource_type="dataset", + uuid=str(uuid4()), + ) + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) + self._assertCloningWithPerms(resource) + + @patch.dict(os.environ, {"ASYNC_SIGNALS": "False"}) + @override_settings(ASYNC_SIGNALS=False) + def test_resource_service_copy_with_perms_dataset_set_default_perms(self): + with self.settings(ASYNC_SIGNALS=False): + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") + files_as_dict, _ = get_files(files) + resource = resource_manager.create( + None, + resource_type=Dataset, + defaults={ + "owner": get_user_model().objects.first(), + "title": "test_copy_with_perms", + "name": "test_copy_with_perms", + "is_approved": True, + "store": "geonode_data", + "subtype": "vector", + "resource_type": "dataset", + "files": files_as_dict.values(), + }, + ) + _perms = { + "users": {"bobby": ["base.add_resourcebase", "base.download_resourcebase"]}, + "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebase"]}, + } + resource.set_permissions(_perms) + # checking that bobby is in the original dataset perms list + self.assertTrue("bobby" in "bobby" in [x.username for x in resource.get_all_level_info().get("users", [])]) + # copying the resource, should remove the perms for bobby + # only the default perms should be available + copy_url = reverse("importer_resource_copy", kwargs={"pk": resource.pk}) + + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.put(copy_url) + self.assertEqual(response.status_code, 200) + + resouce_service_dispatcher.apply((response.json().get("execution_id"),)) + + self.assertEqual("finished", self.client.get(response.json().get("status_url")).json().get("status")) + _resource = Dataset.objects.filter(title__icontains="test_copy_with_perms").last() + self.assertIsNotNone(_resource) + self.assertNotIn("bobby", [x.username for x in _resource.get_all_level_info().get("users", [])]) + self.assertIn("admin", [x.username for x in _resource.get_all_level_info().get("users", [])]) + + def test_resource_service_copy_with_perms_doc(self): + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") + files_as_dict, _ = get_files(files) + resource = Document.objects.create( + owner=get_user_model().objects.get(username="admin"), + subtype="vector", + alternate="geonode:test_copy", + resource_type="document", + uuid=str(uuid4()), + ) + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) + self._assertCloningWithPerms(resource) + + @override_settings(CELERY_TASK_ALWAYS_EAGER=True) + def test_resource_service_copy_with_perms_map(self): + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") + files_as_dict, _ = get_files(files) + resource = Document.objects.create( + owner=get_user_model().objects.get(username="admin"), + alternate="geonode:test_copy", + resource_type="map", + uuid=str(uuid4()), + ) + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) + self._assertCloningWithPerms(resource) + + def _assertCloningWithPerms(self, resource): + # login as bobby + self.assertTrue(self.client.login(username="bobby", password="bob")) + + # bobby cannot copy the resource since he doesnt have all the perms needed + _perms = {"users": {"bobby": ["base.add_resourcebase"]}, "groups": {"anonymous": []}} + resource.set_permissions(_perms) + copy_url = reverse("importer_resource_copy", kwargs={"pk": resource.pk}) + response = self.client.put(copy_url, data={"title": "cloned_resource"}) + self.assertIn(response.status_code, [403, 404]) + # set perms to enable user clone resource + # bobby can copy the resource since he has all the perms needed + _perms = { + "users": {"bobby": ["base.add_resourcebase", "base.download_resourcebase"]}, + "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebae"]}, + } + resource.set_permissions(_perms) + copy_url = reverse("importer_resource_copy", kwargs={"pk": resource.pk}) + response = self.client.put(copy_url, data={"title": "cloned_resource"}) + self.assertEqual(response.status_code, 200) + resource.delete() + + def _get_for_object(self, o, viewname): + url = reverse(viewname, args=[o.id]) + response = self.client.get(url, format="json") + return response.json() + + def _get_for_map(self, viewname): + return self._get_for_object(Map.objects.first(), viewname) + + def test_base_resources_return_download_link_if_document(self): + """ + Ensure we can access the Resource Base list. + """ + doc = Document.objects.first() + + # From resource base API + json = self._get_for_object(doc, "base-resources-detail") + download_url = json.get("resource").get("download_url") + self.assertEqual(build_absolute_uri(doc.download_url), download_url) + + # from documents api + json = self._get_for_object(doc, "documents-detail") + download_url = json.get("document").get("download_url") + self.assertEqual(build_absolute_uri(doc.download_url), download_url) + + def test_base_resources_return_download_link_if_dataset(self): + """ + Ensure we can access the Resource Base list. + """ + _dataset = Dataset.objects.first() + + # From resource base API + json = self._get_for_object(_dataset, "base-resources-detail") + download_url = json.get("resource").get("download_url") + self.assertEqual(_dataset.download_url, download_url) + + # from dataset api + json = self._get_for_object(_dataset, "datasets-detail") + download_url = json.get("dataset").get("download_url") + self.assertEqual(_dataset.download_url, download_url) + + def test_base_resources_dont_return_download_link_if_map(self): + """ + Ensure we can access the Resource Base list. + """ + # From resource base API + json = self._get_for_map("base-resources-detail") + download_url = json.get("resource").get("download_url", None) + self.assertIsNone(download_url) + + # from maps api + json = self._get_for_map("maps-detail") + download_url = json.get("map").get("download_url") + self.assertIsNone(download_url) + + def test_base_resources_return_not_download_links_for_maps(self): + """ + Ensure we can access the Resource Base list. + """ + # From resource base API + json = self._get_for_map("base-resources-detail") + download_url = json.get("resource").get("download_urls", None) + self.assertListEqual([], download_url) + + # from maps api + json = self._get_for_map("maps-detail") + download_url = json.get("map").get("download_urls") + self.assertListEqual([], download_url) + + def test_base_resources_return_download_links_for_documents(self): + """ + Ensure we can access the Resource Base list. + """ + doc = Document.objects.first() + asset = get_default_asset(doc) + handler = asset_handler_registry.get_handler(asset) + expected_payload = [ + {"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe}, + {"ajax_safe": False, "default": False, "url": handler.create_download_url(asset)}, + ] + # From resource base API + json = self._get_for_object(doc, "base-resources-detail") + download_url = json.get("resource").get("download_urls") + self.assertListEqual(expected_payload, download_url) + + # from documents api + json = self._get_for_object(doc, "documents-detail") + download_url = json.get("document").get("download_urls") + self.assertListEqual(expected_payload, download_url) + + def test_base_resources_return_download_links_for_datasets(self): + """ + Ensure we can access the Resource Base list. + """ + _dataset = Dataset.objects.first() + expected_payload = [ + {"url": reverse("dataset_download", args=[_dataset.alternate]), "ajax_safe": True, "default": True} + ] + + # From resource base API + json = self._get_for_object(_dataset, "base-resources-detail") + download_url = json.get("resource").get("download_urls") + self.assertEqual(expected_payload, download_url) + + # from dataset api + json = self._get_for_object(_dataset, "datasets-detail") + download_url = json.get("dataset").get("download_urls") + self.assertEqual(expected_payload, download_url) + + def test_include_linked_resources(self): + dataset = Dataset.objects.first() + doc = Document.objects.first() + map = Map.objects.first() + + for resource, typed_viewname in ( + (dataset, "datasets-detail"), + (doc, "documents-detail"), + (map, "maps-detail"), + ): + for viewname in (typed_viewname, "base-resources-detail"): + for include in (True, False): + url = reverse(viewname, args=[resource.id]) + url = f"{url}{'?include[]=linked_resources' if include else ''}" + response = self.client.get(url, format="json").json() + json = next(iter(response.values())) + if include: + self.assertIn("linked_resources", json, "Missing content") + else: + self.assertNotIn("linked_resources", json, "Unexpected content") + + def test_exclude_all_but_one(self): + dataset = Dataset.objects.first() + doc = Document.objects.first() + map = Map.objects.first() + + for resource, typed_viewname in ( + (dataset, "datasets-detail"), + (doc, "documents-detail"), + (map, "maps-detail"), + ): + for viewname in (typed_viewname, "base-resources-detail"): + for field in ( + "pk", + "title", + "perms", + "links", + "linked_resources", + "data", + "link", + ): # test some random fields + url = reverse(viewname, args=[resource.id]) + url = f"{url}?exclude[]=*&include[]={field}" + response = self.client.get(url, format="json").json() + json = next(iter(response.values())) + + self.assertIn(field, json, "Missing content") + self.assertEqual(1, len(json), f"Only expected content was '{field}', found: {json}") + + def test_presets_base(self): + dataset = Dataset.objects.first() + doc = Document.objects.first() + map = Map.objects.first() + + for resource, typed_viewname in ( + (dataset, "datasets-detail"), + (doc, "documents-detail"), + (map, "maps-detail"), + ): + for viewname in (typed_viewname, "base-resources-detail"): + url = reverse(viewname, args=[resource.id]) + url = f"{url}?api_preset=bare" + response = self.client.get(url, format="json").json() + json = next(iter(response.values())) + self.assertSetEqual( + {"pk", "title"}, + set(json.keys()), + f"Bad json content for object {type(resource)} JSON:{json}", + ) + + def test_api_should_return_all_resources_for_admin(self): + """ + Api whould return all resources even if advertised=False. + """ + url = reverse("base-resources-list") + self.client.login(username="admin", password="admin") + payload = self.client.get(url) + prev_count = payload.json().get("total") + # update all the resource to advertised=False + Dataset.objects.update(advertised=False) + url = reverse("base-resources-list") + payload = self.client.get(url) + new_count = payload.json().get("total") + self.assertEqual(new_count, prev_count) + + Dataset.objects.update(advertised=True) + + def test_api_should_return_advertised_resource_if_anonymous(self): + """ + If anonymous user, only the advertised resoruces whould be returned by the API. + """ + url = reverse("base-resources-list") + payload = self.client.get(url) + prev_count = payload.json().get("total") + # update all the resource to advertised=False + Dataset.objects.update(advertised=False) + url = reverse("base-resources-list") + payload = self.client.get(url) + new_count = payload.json().get("total") + self.assertNotEqual(new_count, prev_count) + + Dataset.objects.update(advertised=True) + + def test_api_should_return_only_the_advertised_false_where_user_is_owner(self): + """ + Api Should return all the resource with advertised=True + And the resource with advertised=False if is owner of it + """ + # defining a new user + test_user_for_api = get_user_model().objects.create(username="test_user_for_api", password="password") + # creating a new resource for the user with advertised=False + dataset = create_single_dataset(name="test_resource_for_api", owner=test_user_for_api, advertised=False) + url = reverse("base-resources-list") + self.client.force_login(test_user_for_api) + payload = self.client.get(f"{url}?limit=1000") + # the uuid of the dataset is in the returned payload + self.assertTrue(dataset.uuid in [k["uuid"] for k in payload.json()["resources"]]) + # bobby is not able to see the dataset belonging to the previous user + self.client.login(username="bobby", password="bob") + payload = self.client.get(url) + self.assertFalse(dataset.uuid in [k["uuid"] for k in payload.json()["resources"]]) + + # cleanup + dataset.delete() + test_user_for_api.delete() + + def test_api_should_filter_by_advertised_param(self): + """ + If anonymous user, only the advertised resoruces whould be returned by the API. + """ + dts = create_single_dataset("advertised_false") + dts.advertised = False + dts.save() + # should show the result based on the logic + url = reverse("base-resources-list") + payload = self.client.get(url) + prev_count = payload.json().get("total") + # the user can see only the advertised resources + self.assertEqual(ResourceBase.objects.filter(advertised=True).count(), prev_count) + + payload = self.client.get(f"{url}?advertised=True") + # so if advertised is True, we dont see the advertised=False resource + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count) + + payload = self.client.get(f"{url}?advertised=False") + # so if advertised is False, we see only the resource with advertised==False + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, 1) + + # if all is requested, we will see all the resources + payload = self.client.get(f"{url}?advertised=all") + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count + 1) + + Dataset.objects.update(advertised=True) + + def test_metadata_uploaded_preserve_can_be_updated(self): + doc = Document.objects.first() + user = get_user_model().objects.get(username="bobby") + url = reverse("base-resources-detail", kwargs={"pk": doc.pk}) + self.assertTrue(self.client.login(username="bobby", password="bob")) + + payload = json.dumps({"metadata_uploaded_preserve": True}) + # should return 403 since bobby doesn't have the perms to update the metadata + # on this resource + response = self.client.patch(url, data=payload, content_type="application/json") + self.assertEqual(403, response.status_code) + doc.refresh_from_db() + # the original value should be kept + self.assertFalse(doc.metadata_uploaded_preserve) + + # let's give to bobby the perms for update the metadata + assign_perm("base.change_resourcebase_metadata", user, doc.get_self_resource()) + + # let's call the API again + response = self.client.patch(url, data=payload, content_type="application/json") + self.assertEqual(200, response.status_code) + doc.refresh_from_db() + # the value should be updated + self.assertTrue(doc.metadata_uploaded_preserve) + self.assertTrue(response.json()["resource"]["metadata_uploaded_preserve"]) + + +class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport): + def setUp(self): + self.layer = create_single_dataset("single_layer") + self.metadata = { + "filter_header": "Foo Filter header", + "field_name": "metadata-name", + "field_label": "this is the help text", + "field_value": "foo", + } + m = ExtraMetadata.objects.create(resource=self.layer, metadata=self.metadata) + self.layer.metadata.add(m) + self.mdata = ExtraMetadata.objects.first() + + def test_get_will_return_the_list_of_extra_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.get(url, content_type="application/json") + self.assertTrue(200, response.status_code) + expected = [{**{"id": self.mdata.id}, **self.metadata}] + self.assertEqual(expected, response.json()) + + def test_put_will_update_the_whole_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + input_metadata = { + "id": self.mdata.id, + "filter_header": "Foo Filter header", + "field_name": "metadata-updated", + "field_label": "this is the help text", + "field_value": "foo", + } + response = self.client.put(url, data=[input_metadata], content_type="application/json") + self.assertTrue(200, response.status_code) + self.assertEqual([input_metadata], response.json()) + + def test_post_will_add_new_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + input_metadata = { + "filter_header": "Foo Filter header", + "field_name": "metadata-updated", + "field_label": "this is the help text", + "field_value": "foo", + } + response = self.client.post(url, data=[input_metadata], content_type="application/json") + self.assertTrue(201, response.status_code) + self.assertEqual(2, len(response.json())) + + def test_delete_will_delete_single_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.delete(url, data=[self.mdata.id], content_type="application/json") + self.assertTrue(200, response.status_code) + self.assertEqual([], response.json()) + + def test_user_without_view_perms_cannot_see_the_endpoint(self): + from geonode.resource.manager import resource_manager + + self.client.login(username="bobby", password="bob") + resource_manager.remove_permissions(self.layer.uuid, instance=self.layer.get_self_resource()) + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.get(url, content_type="application/json") + self.assertTrue(403, response.status_code) + + perm_spec = {"users": {"bobby": ["view_resourcebase"]}, "groups": {}} + self.layer.set_permissions(perm_spec) + url = reverse("base-resources-extra-metadata", args=[self.layer.id]) + response = self.client.get(url, content_type="application/json") + self.assertTrue(200, response.status_code) + + +class TestApiLinkedResources(GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.dataset = create_single_dataset("single_layer") + cls.map = create_single_map("single_map") + cls.doc = create_single_doc("single_doc") + + def test_only_get_method_is_available(self): + url = reverse("base-resources-linked_resources", args=[self.dataset.id]) + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + + response = self.client.delete(url) + self.assertEqual(response.status_code, 403) + + response = self.client.post(url) + self.assertEqual(response.status_code, 403) + + response = self.client.patch(url) + self.assertEqual(response.status_code, 403) + + response = self.client.put(url) + self.assertEqual(response.status_code, 403) + + def test_insert_one_linked_resource(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + self.client.force_login(get_user_model().objects.get(username="admin")) + + response = self.client.post(url, data={"target": [self.map.id]}, content_type="application/json") + + link_connected = LinkedResource.objects.get(source_id=self.doc.id) + + self.assertEqual(response.status_code, 200) + + response_json = response.json() + + self.assertTrue((self.map.id in response_json["success"])) + + self.assertEqual(self.doc.id, link_connected.source_id) + + self.assertEqual(self.map.id, link_connected.target_id) + + def test_insert_linked_resource_invalid_type(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + self.client.force_login(get_user_model().objects.get(username="admin")) + + response = self.client.post(url, data={"target": self.map.id}, content_type="application/json") + self.assertEqual(response.status_code, 400) + + def test_insert_self_as_linked_resource(self): + self.client.force_login(get_user_model().objects.get(username="admin")) + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + # linked resource cannot be linked to itself + response = self.client.post(url, data={"target": [self.doc.id]}, content_type="application/json") + response_json = response.json() + self.assertEqual(response.status_code, 400) + self.assertTrue((self.doc.id in response_json["error"])) + + def test_insert_bad_payload_linked_resource(self): + self.client.force_login(get_user_model().objects.get(username="admin")) + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + # linked resource cannot have an invalid payload + response = self.client.post(url, data={"target_XXX": [self.doc.id]}) + + self.assertEqual(response.status_code, 400) + + def test_insert_existing_linked_resource(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + self.client.force_login(get_user_model().objects.get(username="admin")) + LinkedResource.objects.create(source_id=self.doc.id, target_id=self.doc.id) + + # linked resource cannot be duplicated + response = self.client.post(url, data={"target": [self.doc.id]}, content_type="application/json") + response_json = response.json() + self.assertEqual(response.status_code, 400) + self.assertTrue((self.doc.id in response_json["error"])) + + def test_insert_multiple_linked_resource(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + self.client.force_login(get_user_model().objects.get(username="admin")) + + response = self.client.post( + url, data={"target": [self.map.id, self.dataset.id]}, content_type="application/json" + ) + + list_connected = LinkedResource.objects.filter(source_id=self.doc.id).all() + + list_connected_targets = [linked.target_id for linked in list_connected] + + self.assertEqual(response.status_code, 200) + self.assertEqual(2, len(list_connected)) + response_json = response.json() + self.assertTrue((self.map.id in response_json["success"])) + self.assertTrue((self.dataset.id in response_json["success"])) + self.assertEqual(2, len(response_json["success"])) + + self.assertTrue((self.map.id in list_connected_targets)) + self.assertTrue((self.dataset.id in list_connected_targets)) + + def test_insert_invalid_linked_resource(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + self.client.force_login(get_user_model().objects.get(username="admin")) + + # generate an invalid id + invalid_id = ResourceBase.objects.last().id + 1 + + # make sure id does not exist + invalid_resource = ResourceBase.objects.filter(id=invalid_id).first() + self.assertEqual(None, invalid_resource) + + response = self.client.post(url, data={"target": [invalid_id]}, content_type="application/json") + response_json = response.json() + self.assertEqual(response.status_code, 400) + self.assertTrue((invalid_id in response_json["error"])) + + def test_insert_valid_and_invalid_linked_resource(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + self.client.force_login(get_user_model().objects.get(username="admin")) + + # generate an invalid id + invalid_id = ResourceBase.objects.last().id + 1 + + # make sure id does not exist + invalid_resource = ResourceBase.objects.filter(id=invalid_id).first() + self.assertEqual(None, invalid_resource) + + response = self.client.post(url, data={"target": [invalid_id, self.map.id]}, content_type="application/json") + response_json = response.json() + self.assertEqual(response.status_code, 400) + self.assertTrue((invalid_id in response_json["error"])) + self.assertTrue((self.map.id in response_json["success"])) + + def test_delete_invalid_linked_resource(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + self.client.force_login(get_user_model().objects.get(username="admin")) + + # generate an invalid id + invalid_id = ResourceBase.objects.last().id + 1 + + # make sure id does not exist + invalid_resource = ResourceBase.objects.filter(id=invalid_id).first() + self.assertEqual(None, invalid_resource) + + response = self.client.delete(url, data={"target": [invalid_id]}, content_type="application/json") + response_json = response.json() + self.assertEqual(response.status_code, 400) + self.assertTrue((invalid_id in response_json["error"])) + + def test_delete_linked_resource(self): + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + + self.client.force_login(get_user_model().objects.get(username="admin")) + + LinkedResource.objects.create(source_id=self.doc.id, target_id=self.map.id) + + list_connected = LinkedResource.objects.filter(source_id=self.doc.id).all() + # check count after insertion + self.assertEqual(1, len(list_connected)) + + response = self.client.delete(url, data={"target": [self.map.id]}, content_type="application/json") + response_json = response.json() + self.assertEqual(response.status_code, 200) + # check count after deletion + self.assertEqual(0, len(LinkedResource.objects.filter(source_id=self.doc.id).all())) + self.assertTrue((self.map.id in response_json["success"])) + + def test_delete_not_found_from_linked_resource(self): + self.client.force_login(get_user_model().objects.get(username="admin")) + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + # Make sure there are no linked resource + linked_res = LinkedResource.objects.filter(source_id=self.doc.id).all() + for link in linked_res: + link.delete() + + # try deleting a valid resource but not found in linked res + + response = self.client.delete(url, data={"target": [self.map.id]}, content_type="application/json") + response_json = response.json() + self.assertEqual(response.status_code, 400) + self.assertTrue((self.map.id in response_json["error"])) + + def test_linked_resource_for_document(self): + _d = [] + try: + # data preparation + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.map.id)) + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.dataset.id)) + + # call the API + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + response = self.client.get(url) + + # validation + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assert_linkedres_size(payload, "linked_to", 2) + self.assert_linkedres_contains( + payload, + "linked_to", + ({"pk": self.map.id, "title": self.map.title}, {"pk": self.dataset.id, "title": self.dataset.title}), + ) + self.assert_linkedres_size(payload, "linked_by", 0) + finally: + for d in _d: + d.delete() + + def assert_linkedres_size(self, payload, element: str, expected_size: int): + self.assertEqual( + expected_size, + len(payload[element]), + f"Mismatching payload size of '{element}': exp:{expected_size} found:{payload[element]}", + ) + + def assert_linkedres_contains(self, payload, element: str, expected_elements: Iterable): + res_list = payload[element] + for dikt in expected_elements: + found = False + for res in res_list: + try: + if dikt.items() <= res.items(): + found = True + break + except AttributeError: + self.fail(f"\nError while comparing \n EXPECTED: {dikt}\n FOUND: {res}") + + if not found: + self.fail(f"Elements {dikt} could not be found in output: {payload}") + + def test_linked_resource_for_maps_mixed(self): + _d = [] + try: + # data preparation + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.map.id)) + _d.append( + MapLayer.objects.create( + map=self.map, + dataset=self.dataset, + name=self.dataset.name, + current_style="test_style", + ows_url="https://maps.geosolutionsgroup.com/geoserver/wms", + ) + ) + + # call the API + url = reverse("base-resources-linked_resources", args=[self.map.id]) + response = self.client.get(url) + + # validation + self.assertEqual(response.status_code, 200) + + payload = response.json() + self.assert_linkedres_size(payload, "linked_to", 1) + self.assert_linkedres_contains( + payload, "linked_to", ({"pk": self.dataset.id, "title": self.dataset.title},) + ) + self.assert_linkedres_size(payload, "linked_by", 1) + self.assert_linkedres_contains(payload, "linked_by", ({"pk": self.doc.id, "title": self.doc.title},)) + + finally: + for d in _d: + d.delete() + + def test_linked_resources_for_maps(self): + _m = None + try: + # data preparation + _m = MapLayer.objects.create( + map=self.map, + dataset=self.dataset, + name=self.dataset.name, + current_style="test_style", + ows_url="https://maps.geosolutionsgroup.com/geoserver/wms", + ) + + # call the API + url = reverse("base-resources-linked_resources", args=[self.map.id]) + response = self.client.get(url) + + # validation + self.assertEqual(response.status_code, 200) + + payload = response.json() + self.assert_linkedres_size(payload, "linked_to", 1) + self.assert_linkedres_contains( + payload, "linked_to", ({"pk": self.dataset.id, "title": self.dataset.title},) + ) + self.assert_linkedres_size(payload, "linked_by", 0) + + finally: + if _m: + _m.delete() + + def test_linked_resource_for_dataset(self): + _m = None + try: + # data preparation + _m = MapLayer.objects.create( + map=self.map, + dataset=self.dataset, + name=self.dataset.name, + current_style="test_style", + ows_url="https://maps.geosolutionsgroup.com/geoserver/wms", + ) + + # call the API + url = reverse("base-resources-linked_resources", args=[self.dataset.id]) + response = self.client.get(url) + + # validation + self.assertEqual(response.status_code, 200) + + payload = response.json() + self.assert_linkedres_size(payload, "linked_to", 0) + self.assert_linkedres_size(payload, "linked_by", 1) + self.assert_linkedres_contains(payload, "linked_by", ({"pk": self.map.id, "title": self.map.title},)) + + finally: + if _m: + _m.delete() + + def test_linked_resource_for_datasets_mixed(self): + _d = [] + try: + # data preparation + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.dataset.id)) + _d.append( + MapLayer.objects.create( + map=self.map, + dataset=self.dataset, + name=self.dataset.name, + current_style="test_style", + ows_url="https://maps.geosolutionsgroup.com/geoserver/wms", + ) + ) + + # call the API + url = reverse("base-resources-linked_resources", args=[self.dataset.id]) + response = self.client.get(url) + + # validation + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assert_linkedres_size(payload, "linked_to", 0) + self.assert_linkedres_size(payload, "linked_by", 2) + self.assert_linkedres_contains( + payload, + "linked_by", + ( + {"pk": self.map.id, "title": self.map.title}, + {"pk": self.doc.id, "title": self.doc.title}, + ), + ) + + finally: + for d in _d: + d.delete() + + def test_linked_resource_deprecated_pagination(self): + _d = [] + try: + # data preparation + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.dataset.id)) + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.map.id)) + + # call the API w/ pagination + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + url = f"{url}?page_size=1" + response = self.client.get(url) + + # validation + self.assertEqual(response.status_code, 200) + payload = response.json() + + self.assertIn("WARNINGS", payload, f"Missing WARNINGS element for URL {url}") + self.assertIn("PAGINATION", payload["WARNINGS"], "Missing PAGINATION element") + + # call the API w/o pagination + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + response = self.client.get(url) + + # validation + self.assertEqual(response.status_code, 200) + payload = response.json() + + self.assertNotIn("WARNINGS", payload, "Missing WARNINGS element") + + finally: + for d in _d: + d.delete() + + def test_linked_resource_filter_one_resource_type(self): + _d = [] + try: + # data preparation + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.dataset.id)) + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.map.id)) + resource_type_param = "dataset" + # call api with single resource_type param + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + response = self.client.get(f"{url}?resource_type={resource_type_param}") + + # validation + self.assertEqual(response.status_code, 200) + payload = response.json() + + res_types_orig = resource_type_param.split(",") + res_types_payload = [res["resource_type"] for res in payload["linked_to"]] + for r in res_types_payload: + self.assertTrue(r in res_types_orig) + + finally: + for d in _d: + d.delete() + + def test_linked_resource_filter_multiple_resource_type_linktype(self): + _d = [] + try: + # data preparation + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.dataset.id)) + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.map.id)) + resource_type_param = "map" + link_type = "linked_to" + # call the API w/ both parameters + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + response = self.client.get(f"{url}?resource_type={resource_type_param}&link_type={link_type}") + + # validation + self.assertEqual(response.status_code, 200) + payload = response.json() + + res_types_orig = resource_type_param.split(",") + res_types_payload = [res["resource_type"] for res in payload["linked_to"]] + for type in res_types_payload: + self.assertTrue(type in res_types_orig) + self.assertSetEqual({"linked_to"}, set(payload.keys())) + + finally: + for d in _d: + d.delete() + + def test_linked_resource_filter_multiple_resource_type_without_linktype(self): + _d = [] + try: + # data preparation + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.dataset.id)) + _d.append(LinkedResource.objects.create(source_id=self.doc.id, target_id=self.map.id)) + resource_type_param = "dataset,map" + # call the API w/ resource_type + url = reverse("base-resources-linked_resources", args=[self.doc.id]) + response = self.client.get(f"{url}?resource_type={resource_type_param}") + + # validation + self.assertEqual(response.status_code, 200) + payload = response.json() + + res_types_orig = resource_type_param.split(",") + res_types_payload = [res["resource_type"] for res in payload["linked_to"]] + for type in res_types_payload: + self.assertTrue(type in res_types_orig) + payload_keys = {"linked_by", "linked_to"} + self.assertSetEqual(payload_keys, set(payload.keys())) + + finally: + for d in _d: + d.delete() + + +class TestApiAdditionalBBoxCalculation(GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.dataset = create_single_dataset("single_layer") + cls.map = create_single_map("single_map") + cls.doc = create_single_doc("single_doc") + cls.geoapp = create_single_geoapp("single_geoapp") + cls.bbox = {"extent": {"coords": [6847623, 4776382, 7002886, 4878813], "srid": "EPSG:6875"}} + + def setUp(self): + self.admin = get_user_model().objects.get(username="admin") + + def test_dataset_should_not_update_bbox(self): + self.client.force_login(self.admin) + url = reverse("base-resources-detail", kwargs={"pk": self.dataset.get_self_resource().pk}) + response = self.client.patch( + url, + data=json.dumps({"extent": {"coords": [6847623, 4776382, 7002886, 4878813], "srid": "EPSG:6875"}}), + content_type="application/json", + ) + + llbbox = self.dataset.ll_bbox_polygon + srid = self.dataset.srid + + self.assertEqual(200, response.status_code) + self.dataset.refresh_from_db() + self.assertEqual(self.dataset.ll_bbox_polygon, llbbox) + self.assertEqual(self.dataset.srid, srid) + + def test_map_should_not_update_bbox(self): + self.client.force_login(self.admin) + url = reverse("base-resources-detail", kwargs={"pk": self.map.get_self_resource().pk}) + response = self.client.patch( + url, + data=json.dumps({"extent": {"coords": [6847623, 4776382, 7002886, 4878813], "srid": "EPSG:6875"}}), + content_type="application/json", + ) + + llbbox = self.map.ll_bbox_polygon + srid = self.map.srid + + self.assertEqual(200, response.status_code) + self.map.refresh_from_db() + self.assertEqual(self.map.ll_bbox_polygon, llbbox) + self.assertEqual(self.map.srid, srid) + + def test_document_should_update_bbox(self): + self.client.force_login(self.admin) + url = reverse("base-resources-detail", kwargs={"pk": self.doc.get_self_resource().pk}) + response = self.client.patch( + url, + data=json.dumps({"extent": {"coords": [6847623, 4776382, 7002886, 4878813], "srid": "EPSG:6875"}}), + content_type="application/json", + ) + + self.assertEqual(200, response.status_code) + self.doc.refresh_from_db() + expected = { + "coords": [10.094299016880456, 43.172169804633185, 12.036103612263465, 44.11087068031093], + "srid": "EPSG:4326", + } + resp = response.json()["resource"].get("extent") + self.assertEqual(resp, expected) + self.assertEqual("EPSG:6875", self.doc.srid) + expected = "POLYGON ((10.094299016880456 43.172169804633185, 10.094299016880456 44.11087068031093, 12.036103612263465 44.11087068031093, 12.036103612263465 43.172169804633185, 10.094299016880456 43.172169804633185))" # noqa + self.assertEqual(self.doc.ll_bbox_polygon.wkt, expected) + + def test_geoapp_should_update_bbox(self): + self.client.force_login(self.admin) + url = reverse("base-resources-detail", kwargs={"pk": self.geoapp.get_self_resource().pk}) + response = self.client.patch( + url, + data=json.dumps({"extent": {"coords": [6847623, 4776382, 7002886, 4878813], "srid": "EPSG:6875"}}), + content_type="application/json", + ) + + self.assertEqual(200, response.status_code) + self.geoapp.refresh_from_db() + expected = { + "coords": [10.094299016880456, 43.172169804633185, 12.036103612263465, 44.11087068031093], + "srid": "EPSG:4326", + } + resp = response.json()["resource"].get("extent") + self.assertEqual(resp, expected) + expected = "POLYGON ((10.094299016880456 43.172169804633185, 10.094299016880456 44.11087068031093, 12.036103612263465 44.11087068031093, 12.036103612263465 43.172169804633185, 10.094299016880456 43.172169804633185))" # noqa + self.assertEqual(self.geoapp.ll_bbox_polygon.wkt, expected) + self.assertEqual("EPSG:6875", self.geoapp.srid) + + def test_geoapp_send_invalid_bbox_should_raise_error(self): + self.client.force_login(self.admin) + url = reverse("base-resources-detail", kwargs={"pk": self.geoapp.get_self_resource().pk}) + response = self.client.patch( + url, + data=json.dumps({"extent": {"coords": [6847623, 4776382, 7002886], "srid": "EPSG:6875"}}), + content_type="application/json", + ) + + self.assertEqual(500, response.status_code) + expected = { + "success": False, + "errors": ["The standard bbox provided is invalid"], + "code": "invalid_resource_exception", + } + self.assertDictEqual(expected, response.json()) + + +pycsw_settings_all = settings.PYCSW.copy() + +pycsw_settings_all["FILTER"] = {"resource_type__in": ["dataset", "resourcebase"]} + + +class TestBaseResourceBase(GeoNodeBaseTestSupport): + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.user = get_user_model().objects.get(username="admin") + + def test_simple_resourcebase_can_be_created_by_resourcemanager(self): + self.maxDiff = None + resource = resource_manager.create( + str(uuid4()), resource_type=ResourceBase, defaults={"title": "simple resourcebase", "owner": self.user} + ) + + self.assertIsNotNone(resource) + # minimum resource checks + self.assertEqual("resourcebase", resource.resource_type) + self.assertTrue(resource.link_set.all().exists()) + self.assertEqual("simple resourcebase", resource.title) + self.assertTrue(self.user == resource.owner) + # check if the perms are set + anonymous_group = Group.objects.get(name="anonymous") + perms_expected = { + "users": { + self.user: set( + [ + "delete_resourcebase", + "publish_resourcebase", + "change_resourcebase_permissions", + "view_resourcebase", + "change_resourcebase_metadata", + "change_resourcebase", + ] + ) + }, + "groups": {anonymous_group: set(["view_resourcebase"])}, + } + + actual_perms = resource.get_all_level_info().copy() + self.assertIsNotNone(actual_perms) + self.assertTrue(self.user in actual_perms["users"].keys()) + self.assertTrue(anonymous_group in actual_perms["groups"].keys()) + self.assertSetEqual(perms_expected["users"][self.user], set(actual_perms["users"][self.user])) + self.assertSetEqual(perms_expected["groups"][anonymous_group], set(actual_perms["groups"][anonymous_group])) + + # check if is returned from the API + + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get(reverse("base-resources-list")) + + self.assertTrue(resource.pk in [int(x["pk"]) for x in response.json()["resources"]]) + + # checking csw call + catalogue_post_save(instance=resource, sender=resource.__class__) + # get all records + csw = get_catalogue() + record = csw.get_record(resource.uuid) + self.assertIsNotNone(record) + self.assertEqual(record.identification[0].title, resource.title) + + def test_csw_should_not_return_resourcebase_by_default(self): + resource = resource_manager.create( + str(uuid4()), resource_type=ResourceBase, defaults={"title": "simple resourcebase", "owner": self.user} + ) + dt = resource_manager.create( + str(uuid4()), resource_type=Dataset, defaults={"title": "simple dataset", "owner": self.user} + ) + + self.assertTrue(ResourceBase.objects.filter(pk=resource.pk).exists()) + self.assertTrue(ResourceBase.objects.filter(pk=dt.pk).exists()) + + request = self.__csw_request_factory() + + response = csw_global_dispatch(request) + root = etree.fromstring(response.content) + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 + self.assertEqual(1, returned_results) + + @override_settings(PYCSW=pycsw_settings_all) + def test_csw_should_return_resourcebase_if_defined_in_settings(self): + resource = resource_manager.create( + str(uuid4()), resource_type=ResourceBase, defaults={"title": "simple resourcebase", "owner": self.user} + ) + dt = resource_manager.create( + str(uuid4()), resource_type=Dataset, defaults={"title": "simple dataset", "owner": self.user} + ) + + self.assertTrue(ResourceBase.objects.filter(pk=resource.pk).exists()) + self.assertTrue(ResourceBase.objects.filter(pk=dt.pk).exists()) + + request = self.__csw_request_factory() + + response = csw_global_dispatch(request) + root = etree.fromstring(response.content) + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 + self.assertEqual(2, returned_results) + + @staticmethod + def __csw_request_factory(): + from django.contrib.auth.models import AnonymousUser + + factory = RequestFactory() + url = "http://localhost:8000/catalogue/csw?request=GetRecords" + url += "&service=CSW&version=2.0.2&outputschema=http%3A%2F%2Fwww.isotc211.org%2F2005%2Fgmd" + url += "&elementsetname=brief&typenames=csw:Record&resultType=results" + request = factory.get(url) + + request.user = AnonymousUser() + return request diff --git a/geonode/base/api/urls.py b/geonode/base/api/urls.py new file mode 100644 index 0000000..69121b9 --- /dev/null +++ b/geonode/base/api/urls.py @@ -0,0 +1,31 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.api.urls import router + +from . import views + +router.register(r"groups", views.GroupViewSet, "group-profiles") +router.register(r"resources", views.ResourceBaseViewSet, "base-resources") +router.register(r"owners", views.OwnerViewSet, "owners") +router.register(r"categories", views.TopicCategoryViewSet, "categories") +router.register(r"keywords", views.HierarchicalKeywordViewSet, "keywords") +router.register(r"tkeywords", views.ThesaurusKeywordViewSet, "tkeywords") +router.register(r"regions", views.RegionViewSet, "regions") + +urlpatterns = [] diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py new file mode 100644 index 0000000..8a797f0 --- /dev/null +++ b/geonode/base/api/views.py @@ -0,0 +1,1436 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import ast +import functools +import json +import re + +from uuid import uuid4 +from urllib.parse import urljoin, urlparse +from PIL import Image + +from django.apps import apps +from django.core.validators import URLValidator +from django.shortcuts import get_object_or_404 +from django.urls import reverse +from django.conf import settings +from django.db.models import Subquery, QuerySet +from django.http.request import QueryDict +from django.contrib.auth import get_user_model + +from drf_spectacular.utils import extend_schema +from dynamic_rest.viewsets import DynamicModelViewSet, WithDynamicViewSetMixin +from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter + +from oauth2_provider.contrib.rest_framework import OAuth2Authentication + +from rest_framework import status +from rest_framework.exceptions import ValidationError +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.parsers import JSONParser, MultiPartParser +from rest_framework.views import APIView +from rest_framework.viewsets import GenericViewSet +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin +from rest_framework.permissions import AllowAny, IsAuthenticated, IsAuthenticatedOrReadOnly +from rest_framework.authentication import SessionAuthentication, BasicAuthentication + +from geonode.maps.models import Map +from geonode.layers.models import Dataset +from geonode.favorite.models import Favorite +from geonode.base.models import Configuration, ExtraMetadata, LinkedResource +from geonode.thumbs.exceptions import ThumbnailError +from geonode.thumbs.thumbnails import create_thumbnail +from geonode.thumbs.utils import _decode_base64, BASE64_PATTERN +from geonode.groups.conf import settings as groups_settings +from geonode.base.models import HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword +from geonode.base.api.filters import ( + DynamicSearchFilter, + ExtentFilter, + FacetVisibleResourceFilter, + FavoriteFilter, + TKeywordsFilter, +) +from geonode.groups.models import GroupProfile +from geonode.security.permissions import get_compact_perms_list, PermSpec, PermSpecCompact +from geonode.security.utils import ( + get_visible_resources, + get_resources_with_perms, + get_user_visible_groups, +) + +from geonode.resource.models import ExecutionRequest +from geonode.resource.api.tasks import resouce_service_dispatcher +from geonode.resource.manager import resource_manager + + +from geonode.base.api.mixins import AdvertisedListMixin +from .permissions import ( + IsOwnerOrAdmin, + IsManagerEditOrAdmin, + ResourceBasePermissionsFilter, + UserHasPerms, +) + +from .serializers import ( + FavoriteSerializer, + PermSpecSerialiazer, + GroupProfileSerializer, + ResourceBaseSerializer, + ResourceBaseTypesSerializer, + OwnerSerializer, + HierarchicalKeywordSerializer, + TopicCategorySerializer, + RegionSerializer, + ThesaurusKeywordSerializer, + ExtraMetadataSerializer, + LinkedResourceSerializer, +) +from geonode.people.api.serializers import UserSerializer +from .pagination import GeoNodeApiPagination +from geonode.base.utils import validate_extra_metadata + +import logging + +logger = logging.getLogger(__name__) + + +class GroupViewSet(DynamicModelViewSet): + """ + API endpoint that allows gropus to be viewed or edited. + """ + + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [ + IsAuthenticatedOrReadOnly, + IsManagerEditOrAdmin, + ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] + serializer_class = GroupProfileSerializer + pagination_class = GeoNodeApiPagination + + def get_queryset(self): + """ + Filters the public groups and private ones the current user is member of. + """ + metadata_author_groups = get_user_visible_groups(self.request.user, include_public_invite=True) + if not isinstance(metadata_author_groups, list): + metadata_author_groups = list(metadata_author_groups.all()) + queryset = GroupProfile.objects.filter(id__in=[_g.id for _g in metadata_author_groups]) + return queryset.order_by("title") + + @extend_schema( + methods=["get"], + responses={200: UserSerializer(many=True)}, + description="API endpoint allowing to retrieve the Group members.", + ) + @action(detail=True, methods=["get"]) + def members(self, request, pk=None): + group = self.get_object() + members = get_user_model().objects.filter(id__in=group.member_queryset().values_list("user", flat=True)) + return Response(UserSerializer(embed=True, many=True).to_representation(members)) + + @extend_schema( + methods=["get"], + responses={200: UserSerializer(many=True)}, + description="API endpoint allowing to retrieve the Group managers.", + ) + @action(detail=True, methods=["get"]) + def managers(self, request, pk=None): + group = self.get_object() + managers = group.get_managers() + return Response(UserSerializer(embed=True, many=True).to_representation(managers)) + + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the Group specific resources.", + ) + @action(detail=True, methods=["get"]) + def resources(self, request, pk=None): + group = self.get_object() + resources = group.resources() + paginator = GeoNodeApiPagination() + paginator.page_size = request.GET.get("page_size", 10) + result_page = paginator.paginate_queryset(list(resources), request) + serializer = ResourceBaseSerializer(result_page, embed=True, many=True, context={"request": request}) + return paginator.get_paginated_response({"resources": serializer.data}) + + +class RegionViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + """ + API endpoint that lists regions. + """ + + permission_classes = [ + AllowAny, + ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, FacetVisibleResourceFilter] + queryset = Region.objects.all() + serializer_class = RegionSerializer + pagination_class = GeoNodeApiPagination + + +class HierarchicalKeywordViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + """ + API endpoint that lists hierarchical keywords. + """ + + def get_queryset(self): + resource_keywords = HierarchicalKeyword.resource_keywords_tree(self.request.user) + + def _get_kw_hrefs(keywords, slugs: list = []): + for obj in keywords: + if obj.get("tags", []): + slugs.append(obj.get("href")) + _get_kw_hrefs(obj.get("nodes", []), slugs) + return slugs + + slugs = _get_kw_hrefs(resource_keywords) + return HierarchicalKeyword.objects.filter(slug__in=slugs) + + permission_classes = [ + AllowAny, + ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] + serializer_class = HierarchicalKeywordSerializer + pagination_class = GeoNodeApiPagination + + +class ThesaurusKeywordViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + """ + API endpoint that lists Thesaurus keywords. + """ + + permission_classes = [ + AllowAny, + ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] + queryset = ThesaurusKeyword.objects.all() + serializer_class = ThesaurusKeywordSerializer + pagination_class = GeoNodeApiPagination + + +class TopicCategoryViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + """ + API endpoint that lists categories. + """ + + permission_classes = [ + AllowAny, + ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, FacetVisibleResourceFilter] + queryset = TopicCategory.objects.all() + serializer_class = TopicCategorySerializer + pagination_class = GeoNodeApiPagination + + +class OwnerViewSet(WithDynamicViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + """ + API endpoint that lists all possible owners. + """ + + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [ + AllowAny, + ] + filter_backends = [DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter] + serializer_class = OwnerSerializer + pagination_class = GeoNodeApiPagination + + def get_queryset(self): + """ + Filter users with at least one resource + """ + queryset = get_user_model().objects.exclude(pk=-1) + filter_options = {} + if self.request.query_params: + filter_options = { + "type_filter": self.request.query_params.get("type"), + "title_filter": self.request.query_params.get("title__icontains"), + } + queryset = queryset.filter( + id__in=Subquery(get_resources_with_perms(self.request.user, filter_options).values("owner")) + ) + return queryset.order_by("username") + + +class ApiPresetsInitializer(APIView): + """ + Replaces the `api_preset` query params with the configured params + """ + + def initialize_request(self, request, *args, **kwargs): + self.replace_presets(request) + return super().initialize_request(request, *args, **kwargs) + + def replace_presets(self, request): + # we must make the GET mutable since in the filters, some queryparams are popped + request.GET._mutable = True + try: + for preset_name in request.GET.pop("api_preset", []): + presets = settings.REST_API_PRESETS.get(preset_name, None) + if not presets: + logger.info(f'Preset "{preset_name}" is not defined') # maybe return 404? + return + for param_name in presets.keys(): + for param_value in presets.get(param_name): + if param_value not in request.GET.get(param_name, []): + request.GET.appendlist(param_name, param_value) + finally: + request.GET._mutable = False + + +class ResourceBaseViewSet(ApiPresetsInitializer, DynamicModelViewSet, AdvertisedListMixin): + """ + API endpoint that allows base resources to be viewed or edited. + """ + + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms] + filter_backends = [ + TKeywordsFilter, + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + ExtentFilter, + ResourceBasePermissionsFilter, + FavoriteFilter, + ] + queryset = ResourceBase.objects.all().order_by("-created") + serializer_class = ResourceBaseSerializer + pagination_class = GeoNodeApiPagination + + def _filtered(self, request, filter): + paginator = GeoNodeApiPagination() + paginator.page_size = request.GET.get("page_size", 10) + resources = get_resources_with_perms(request.user).filter(**filter) + result_page = paginator.paginate_queryset(resources, request) + serializer = ResourceBaseSerializer(result_page, embed=True, many=True) + return paginator.get_paginated_response({"resources": serializer.data}) + + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the approved Resources.", + ) + @action(detail=False, methods=["get"]) + def approved(self, request, *args, **kwargs): + return self._filtered(request, {"is_approved": True}) + + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the published Resources.", + ) + @action(detail=False, methods=["get"]) + def published(self, request, *args, **kwargs): + return self._filtered(request, {"is_published": True}) + + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve the featured Resources.", + ) + @action(detail=False, methods=["get"]) + def featured(self, request, *args, **kwargs): + return self._filtered(request, {"featured": True}) + + @extend_schema( + methods=["get"], + responses={200: FavoriteSerializer(many=True)}, + description="API endpoint allowing to retrieve the favorite Resources.", + ) + @action( + detail=False, + methods=["get"], + permission_classes=[ + IsAuthenticated, + ], + ) + def favorites(self, request, pk=None, *args, **kwargs): + paginator = GeoNodeApiPagination() + paginator.page_size = request.GET.get("page_size", 10) + favorites = Favorite.objects.favorites_for_user(user=request.user) + result_page = paginator.paginate_queryset(favorites, request) + serializer = FavoriteSerializer(result_page, embed=True, many=True) + return paginator.get_paginated_response({"favorites": serializer.data}) + + @extend_schema( + methods=["post", "delete"], + responses={200: FavoriteSerializer(many=True)}, + description="API endpoint allowing to retrieve the favorite Resources.", + ) + @action(detail=True, methods=["post", "delete"], permission_classes=[IsAuthenticated]) + def favorite(self, request, pk=None, *args, **kwargs): + resource = self.get_object() + user = request.user + + if request.method == "POST": + try: + Favorite.objects.get(user=user, object_id=resource.pk) + return Response({"message": "Resource is already in favorites"}, status=400) + except Favorite.DoesNotExist: + Favorite.objects.create_favorite(resource, user) + return Response({"message": "Successfuly added resource to favorites"}, status=201) + + if request.method == "DELETE": + try: + Favorite.objects.get(user=user, object_id=resource.pk).delete() + return Response({"message": "Successfuly removed resource from favorites"}, status=200) + except Favorite.DoesNotExist: + return Response({"message": "Resource not in favorites"}, status=404) + + @extend_schema( + methods=["get"], + responses={200: ResourceBaseTypesSerializer()}, + description=""" + Returns the list of available ResourceBase polymorphic_ctypes. + + the mapping looks like: + ``` + { + "resource_types":[ + { + "name": "layer", + "count": + }, + { + "name": "map", + "count": + }, + { + "name": "document", + "count": + }, + { + "name": "geostory", + "count": + } + ] + } + ``` + """, + ) + @action(detail=False, methods=["get"]) + def resource_types(self, request, *args, **kwargs): + def _to_compact_perms_list( + allowed_perms: dict, resource_type: str, resource_subtype: str, compact_perms_labels: dict = {} + ) -> list: + _compact_perms_list = {} + for _k, _v in allowed_perms.items(): + _is_owner = _k not in ["anonymous", groups_settings.REGISTERED_MEMBERS_GROUP_NAME] + _is_none_allowed = not _is_owner + _compact_perms_list[_k] = get_compact_perms_list( + _v, resource_type, resource_subtype, _is_owner, _is_none_allowed, compact_perms_labels + ) + return _compact_perms_list + + resource_types = [] + _types = [] + _allowed_perms = {} + for _model in apps.get_models(): + if _model.__name__ == "ResourceBase": + for _m in _model.__subclasses__(): + if _m.__name__.lower() not in ["service"]: + _types.append(_m.__name__.lower()) + _allowed_perms[_m.__name__.lower()] = { + "perms": _m.allowed_permissions, + "compact": _to_compact_perms_list( + _m.allowed_permissions, + _m.__name__.lower(), + _m.__name__.lower(), + _m.compact_permission_labels, + ), + } + + if settings.GEONODE_APPS_ENABLE and "geoapp" in _types: + _types.remove("geoapp") + if hasattr(settings, "CLIENT_APP_LIST") and settings.CLIENT_APP_LIST: + _types += settings.CLIENT_APP_LIST + else: + from geonode.geoapps.models import GeoApp + + geoapp_types = [x for x in GeoApp.objects.values_list("resource_type", flat=True).all().distinct()] + _types += geoapp_types + + if hasattr(settings, "CLIENT_APP_ALLOWED_PERMS_LIST") and settings.CLIENT_APP_ALLOWED_PERMS_LIST: + for _type in settings.CLIENT_APP_ALLOWED_PERMS_LIST: + for _type_name, _type_perms in _type.items(): + _compact_permission_labels = {} + if hasattr(settings, "CLIENT_APP_COMPACT_PERM_LABELS"): + _compact_permission_labels = settings.CLIENT_APP_COMPACT_PERM_LABELS.get(_type_name, {}) + _allowed_perms[_type_name] = { + "perms": _type_perms, + "compact": _to_compact_perms_list( + _type_perms, _type_name, _type_name, _compact_permission_labels + ), + } + else: + from geonode.geoapps.models import GeoApp + + for _m in GeoApp.objects.filter(resource_type__in=_types).iterator(): + if hasattr(_m, "resource_type") and _m.resource_type and _m.resource_type not in _allowed_perms: + _allowed_perms[_m.resource_type] = { + "perms": _m.allowed_permissions, + "compact": _to_compact_perms_list( + _m.allowed_permissions, _m.resource_type, _m.subtype, _m.compact_permission_labels + ), + } + + for _type in _types: + resource_types.append( + { + "name": _type, + "count": get_resources_with_perms(request.user).filter(resource_type=_type).count(), + "allowed_perms": _allowed_perms[_type] if _type in _allowed_perms else [], + } + ) + return Response({"resource_types": resource_types}) + + @extend_schema( + methods=["get", "put", "patch", "delete"], + request=PermSpecSerialiazer(), + responses={200: None}, + description=""" + Sets an object's the permission levels based on the perm_spec JSON. + + the mapping looks like: + ``` + { + 'users': { + 'AnonymousUser': ['view'], + : ['perm1','perm2','perm3'], + : ['perm1','perm2','perm3'] + ... + }, + 'groups': { + : ['perm1','perm2','perm3'], + : ['perm1','perm2','perm3'], + ... + } + } + ``` + """, + ) + @action( + detail=True, + url_path="permissions", # noqa + url_name="perms-spec", + methods=["get", "put", "patch", "delete"], + permission_classes=[IsAuthenticated], + ) + def resource_service_permissions(self, request, pk, *args, **kwargs): + """Instructs the Async dispatcher to execute a 'DELETE' or 'UPDATE' on the permissions of a valid 'uuid' + + - GET input_params: { + id: "" + } + + - DELETE input_params: { + id: "" + } + + - PUT input_params: { + id: "" + owner: str = None + permissions: dict = {} + created: bool = False + } + + - output_params: { + output: { + uuid: "" + } + } + + - output: { + "status": "ready", + "execution_id": "", + "status_url": "http://localhost:8000/api/v2/resource-service/execution-status/" + } + + Sample Requests: + - Removes all the permissions (except owner and admin ones) from a Resource: + curl -v -X DELETE -u admin:admin -H "Content-Type: application/json" http://localhost:8000/api/v2/resources//permissions + + - Changes the owner of a Resource: + curl -u admin:admin --location --request PUT 'http://localhost:8000/api/v2/resources//permissions' \ + --header 'Content-Type: application/json' \ + --data-raw '{"groups": [],"organizations": [],"users": [{"id": 1001,"permissions": "owner"}]}' + + - Assigns View permissions to some users: + curl -u admin:admin --location --request PUT 'http://localhost:8000/api/v2/resources//permissions' \ + --header 'Content-Type: application/json' \ + --data-raw '{"groups": [],"organizations": [],"users": [{"id": 1000,"permissions": "view"}]}' + + - Assigns View permissions to anyone: + curl -u admin:admin --location --request PUT 'http://localhost:8000/api/v2/resources//permissions' \ + --header 'Content-Type: application/json' \ + --data-raw '{"groups": [],"organizations": [],"users": [{"id": -1,"permissions": "view"}]}' + + - Assigns View permissions to anyone and edit permissions to a Group on a Dataset: + curl -u admin:admin --location --request PUT 'http://localhost:8000/api/v2/resources//permissions' \ + --header 'Content-Type: application/json' \ + --data-raw '{"groups": [{"id": 1,"permissions": "manage"}],"organizations": [],"users": [{"id": -1,"permissions": "view"}]}' + + """ + config = Configuration.load() + resource = get_object_or_404(ResourceBase, pk=pk) + _user_can_manage = request.user.has_perm("change_resourcebase_permissions", resource.get_self_resource()) + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not _user_can_manage + ): + return Response(status=status.HTTP_403_FORBIDDEN) + try: + perms_spec = PermSpec(resource.get_all_level_info(), resource) + request_params = request.data + if request.method == "GET": + return Response(perms_spec.compact) + elif request.method == "DELETE": + _exec_request = ExecutionRequest.objects.create( + user=request.user, + func_name="remove_permissions", + geonode_resource=resource, + action="permissions", + input_params={"uuid": request_params.get("uuid", resource.uuid)}, + ) + elif request.method == "PUT": + perms_spec_compact = PermSpecCompact(request.data, resource) + _exec_request = ExecutionRequest.objects.create( + user=request.user, + func_name="set_permissions", + geonode_resource=resource, + action="permissions", + input_params={ + "uuid": request_params.get("uuid", resource.uuid), + "owner": request_params.get("owner", resource.owner.username), + "permissions": perms_spec_compact.extended, + "created": request_params.get("created", False), + }, + ) + elif request.method == "PATCH": + perms_spec_compact_patch = PermSpecCompact(request.data, resource) + perms_spec_compact_resource = PermSpecCompact(perms_spec.compact, resource) + perms_spec_compact_resource.merge(perms_spec_compact_patch) + _exec_request = ExecutionRequest.objects.create( + user=request.user, + func_name="set_permissions", + geonode_resource=resource, + action="permissions", + input_params={ + "uuid": request_params.get("uuid", resource.uuid), + "owner": request_params.get("owner", resource.owner.username), + "permissions": perms_spec_compact_resource.extended, + "created": request_params.get("created", False), + }, + ) + resouce_service_dispatcher.apply_async(args=(str(_exec_request.exec_id),), expiration=30) + return Response( + { + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + logger.exception(e) + return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) + + @extend_schema( + methods=["post"], + responses={200}, + description="API endpoint allowing to set the thumbnail url for an existing dataset.", + ) + @action( + detail=False, + url_path="(?P\d+)/set_thumbnail_from_bbox", # noqa + url_name="set-thumb-from-bbox", + methods=["post"], + permission_classes=[IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], + ) + def set_thumbnail_from_bbox(self, request, resource_id, *args, **kwargs): + import traceback + from django.utils.datastructures import MultiValueDictKeyError + + try: + resource = ResourceBase.objects.get(id=ast.literal_eval(resource_id)) + + map_thumb_from_bbox = False + if isinstance(resource.get_real_instance(), Map): + map_thumb_from_bbox = True + + if not isinstance(resource.get_real_instance(), (Dataset, Map)): + raise NotImplementedError("Not implemented: Endpoint available only for Dataset and Maps") + + request_body = request.data if request.data else json.loads(request.body) + try: + # bbox format: (xmin, xmax, ymin, ymax) + bbox = request_body["bbox"] + [request_body["srid"]] + zoom = request_body.get("zoom", None) + except MultiValueDictKeyError: + for _k, _v in request_body.items(): + request_body = json.loads(_k) + break + bbox = request_body["bbox"] + [request_body["srid"]] + zoom = request_body.get("zoom", None) + + thumbnail_url = create_thumbnail( + resource.get_real_instance(), + bbox=bbox, + background_zoom=zoom, + overwrite=True, + map_thumb_from_bbox=map_thumb_from_bbox, + ) + return Response( + {"message": "Thumbnail correctly created.", "success": True, "thumbnail_url": thumbnail_url}, status=200 + ) + except ResourceBase.DoesNotExist: + traceback.print_exc() + logger.error(f"Resource selected with id {resource_id} does not exists") + return Response( + data={"message": f"Resource selected with id {resource_id} does not exists", "success": False}, + status=404, + exception=True, + ) + except NotImplementedError as e: + traceback.print_exc() + logger.error(e) + return Response(data={"message": e.args[0], "success": False}, status=405, exception=True) + except ThumbnailError as e: + traceback.print_exc() + logger.error(e) + return Response(data={"message": e.args[0], "success": False}, status=500, exception=True) + except Exception as e: + traceback.print_exc() + logger.error(e) + return Response(data={"message": e.args[0], "success": False}, status=500, exception=True) + + @extend_schema( + methods=["post"], responses={200}, description="Instructs the Async dispatcher to execute a 'CREATE' operation." + ) + @action( + detail=False, + url_path="create/(?P\w+)", # noqa + url_name="resource-service-create", + methods=["post"], + permission_classes=[IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], + ) + def resource_service_create(self, request, resource_type: str = None, *args, **kwargs): + """Instructs the Async dispatcher to execute a 'CREATE' operation + **WARNING**: This will create an empty dataset; if you need to upload a resource to GeoNode, consider using the endpoint "ingest" instead + + - POST input_params: { + uuid: "", + defaults: "{\"owner\":\"\",}", # WARNING: 'owner' is mandatory + resource_type: "']>" + } + + - output_params: { + output: + } + + - output: { + "status": "ready", + "execution_id": "", + "status_url": "http://localhost:8000/api/v2/resource-service/execution-status/" + } + + Sample Request: + + 1. curl -v -X POST -u admin:admin -H "Content-Type: application/json" -d 'defaults={"owner":"admin","title":"pippo"}' + http://localhost:8000/api/v2/resources/create/dataset + OUTPUT: { + "status": "ready", + "execution_id": "90ca670d-df60-44b6-b358-d792c6aecc58", + "status_url": "http://localhost:8000/api/v2/resource-service/execution-status/90ca670d-df60-44b6-b358-d792c6aecc58" + } + + 2. curl -v -X GET -u admin:admin http://localhost:8000/api/v2/resource-service/execution-status/90ca670d-df60-44b6-b358-d792c6aecc58 + OUTPUT: { + "user": "admin", + "status": "finished", + "func_name": "create", + "created": "2021-07-22T15:32:09.096075Z", + "finished": "2021-07-22T15:32:26.936683Z", + "last_updated": "2021-07-22T15:32:09.096129Z", + "input_params": { + "uuid": "fa404f64-eb01-11eb-8f91-00155d41f2fb", + "defaults": "{\"owner\":\"admin\",\"title\":\"pippo\"}", + "resource_type": "dataset" + }, + "output_params": { + "output": { + "uuid": "fa404f64-eb01-11eb-8f91-00155d41f2fb" + } + } + } + """ + config = Configuration.load() + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or not request.user.has_perm("base.add_resourcebase") + ): + return Response(status=status.HTTP_403_FORBIDDEN) + try: + request_params = self._get_request_params(request) + uuid = request_params.get("uuid", str(uuid4())) + resource_filter = ResourceBase.objects.filter(uuid=uuid) + + _exec_request = ExecutionRequest.objects.create( + user=request.user, + func_name="create", + geonode_resource=resource_filter.get() if resource_filter.exists() else None, + action="create", + input_params={ + "uuid": uuid, + "resource_type": resource_type, + "defaults": request_params.get("defaults", f'{{"owner":"{request.user.username}"}}'), + }, + ) + resouce_service_dispatcher.apply_async(args=(str(_exec_request.exec_id),), expiration=30) + return Response( + { + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + logger.exception(e) + return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) + + @extend_schema( + methods=["delete"], + responses={200}, + description="Instructs the Async dispatcher to execute a 'DELETE' operation over a valid 'uuid'.", + ) + @action( + detail=True, + url_path="delete", # noqa + url_name="resource-service-delete", + methods=["delete"], + permission_classes=[IsAuthenticated, UserHasPerms], + ) + def resource_service_delete(self, request, pk, *args, **kwargs): + """Instructs the Async dispatcher to execute a 'DELETE' operation over a valid 'uuid' + + - DELETE input_params: { + id: "" + } + + - output_params: { + output: + } + + - output: { + "status": "ready", + "execution_id": "", + "status_url": "http://localhost:8000/api/v2/resource-service/execution-status/" + } + + Sample request: + + 1. curl -v -X DELETE -u admin:admin http://localhost:8000/api/v2/resources//delete + OUTPUT: { + "status":"ready", + "execution_id":"7ed0b141-cf85-434f-bbfb-c02447a5221b", + "status_url":"http://localhost:8000/api/v2/resource-service/execution-status/7ed0b141-cf85-434f-bbfb-c02447a5221b" + } + + 2. curl -v -X GET -u admin:admin http://localhost:8000/api/v2/resource-service/execution-status/7ed0b141-cf85-434f-bbfb-c02447a5221b + OUTPUT: { + "user":"admin", + "status":"finished", + "func_name":"delete", + "created":"2021-07-19T14:09:59.930619Z", + "finished":"2021-07-19T14:10:00.054915Z", + "last_updated":"2021-07-19T14:09:59.930647Z", + "input_params":{"uuid":"1234"}, + "output_params":{"output":0} + } + """ + config = Configuration.load() + resource = get_object_or_404(ResourceBase, pk=pk) + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not request.user.has_perm("delete_resourcebase", resource.get_self_resource()) + ): + return Response(status=status.HTTP_403_FORBIDDEN) + try: + _exec_request = ExecutionRequest.objects.create( + user=request.user, + func_name="delete", + action="delete", + geonode_resource=resource, + input_params={"uuid": resource.uuid}, + ) + resouce_service_dispatcher.apply_async(args=(str(_exec_request.exec_id),), expiration=30) + return Response( + { + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + logger.exception(e) + return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) + + @extend_schema( + methods=["put"], + responses={200}, + description="Instructs the Async dispatcher to execute a 'UPDATE' operation over a valid 'uuid'.", + ) + @action( + detail=True, + url_path="update", # noqa + url_name="resource-service-update", + methods=["put"], + permission_classes=[IsAuthenticated, UserHasPerms], + ) + def resource_service_update(self, request, pk, *args, **kwargs): + """Instructs the Async dispatcher to execute a 'UPDATE' operation over a valid 'uuid' + + - PUT input_params: { + id: "" + xml_file: str = None + metadata_uploaded: bool = False + vals: dict = {} + regions: list = [] + keywords: list = [] + custom: dict = {} + notify: bool = True + } + + - output_params: { + output: { + uuid: "" + } + } + + - output: { + "status": "ready", + "execution_id": "", + "status_url": "http://localhost:8000/api/v2/resource-service/execution-status/" + } + + Sample Request: + + 1. curl -v -X PUT -u admin:admin -H "Content-Type: application/json" -d 'vals={"title":"pippo"}' http://localhost:8000/api/v2/resources//update + OUTPUT: { + "status":"ready", + "execution_id":"08846e84-eae4-11eb-84be-00155d41f2fb", + "status_url":"http://localhost:8000/api/v2/resource-service/execution-status/08846e84-eae4-11eb-84be-00155d41f2fb" + } + + 2. curl -v -X GET -u admin:admin http://localhost:8000/api/v2/resource-service/execution-status/08846e84-eae4-11eb-84be-00155d41f2fb + OUTPUT: { + "user": "admin", + "status": "finished", + "func_name": "update", + "created": "2021-07-22T14:42:56.284740Z", + "finished": "2021-07-22T14:43:01.813971Z", + "last_updated": "2021-07-22T14:42:56.284797Z", + "input_params": { + "uuid": "ee11541c-eaee-11eb-942c-00155d41f2fb", + "vals": "{\"title\":\"pippo\"}", + "custom": {}, + "notify": true, + "regions": [], + "keywords": [], + "xml_file": null, + "metadata_uploaded": false + }, + "output_params": { + "output": { + "uuid": "ee11541c-eaee-11eb-942c-00155d41f2fb" + } + } + } + + Sample Request with more parameters: + + 1. curl -v -X PUT -u admin:admin -H "Content-Type: application/json" -d 'vals={"title":"pippo"}' -d 'metadata_uploaded=true' -d 'keywords=["k1", "k2", "k3"]' + http://localhost:8000/api/v2/resources//update + """ + config = Configuration.load() + resource = get_object_or_404(ResourceBase, pk=pk) + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not request.user.has_perm("change_resourcebase", resource.get_self_resource()) + ): + return Response(status=status.HTTP_403_FORBIDDEN) + try: + request_params = self._get_request_params(request=request) + _exec_request = ExecutionRequest.objects.create( + user=request.user, + func_name="update", + geonode_resource=resource, + action="update", + input_params={ + "uuid": request_params.get("uuid", resource.uuid), + "xml_file": request_params.get("xml_file", None), + "metadata_uploaded": request_params.get("metadata_uploaded", False), + "vals": request_params.get("vals", "{}"), + "regions": request_params.get("regions", "[]"), + "keywords": request_params.get("keywords", "[]"), + "custom": request_params.get("custom", "{}"), + "notify": request_params.get("notify", True), + }, + ) + resouce_service_dispatcher.apply_async(args=(str(_exec_request.exec_id),), expiration=30) + return Response( + { + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + logger.exception(e) + return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) + + @extend_schema( + methods=["put"], + responses={200}, + description="Instructs the Async dispatcher to execute a 'COPY' operation over a valid 'uuid'.", + ) + @action( + detail=True, + url_path="copy", # noqa + url_name="resource-service-copy", + methods=["put"], + permission_classes=[ + IsAuthenticated, + UserHasPerms( + perms_dict={ + "dataset": {"PUT": ["base.add_resourcebase", "base.download_resourcebase"], "rule": all}, + "document": {"PUT": ["base.add_resourcebase", "base.download_resourcebase"], "rule": all}, + "default": {"PUT": ["base.add_resourcebase"]}, + } + ), + ], + ) + def resource_service_copy(self, request, pk, *args, **kwargs): + """Instructs the Async dispatcher to execute a 'COPY' operation over a valid 'pk' + + - PUT input_params: { + instance: "" + owner: ">" + defaults: dict = {} + } + + - output_params: { + output: { + uuid: "" + } + } + + - output: { + "status": "ready", + "execution_id": "", + "status_url": "http://localhost:8000/api/v2/resource-service/execution-status/" + } + + Sample Request: + + 1. curl -v -X PUT -u admin:admin -H "Content-Type: application/json" -d 'defaults={"title":"pippo"}' http://localhost:8000/api/v2/resources//copy + OUTPUT: { + "status":"ready", + "execution_id":"08846e84-eae4-11eb-84be-00155d41f2fb", + "status_url":"http://localhost:8000/api/v2/resource-service/execution-status/08846e84-eae4-11eb-84be-00155d41f2fb" + } + + 2. curl -v -X GET -u admin:admin http://localhost:8000/api/v2/resource-service/execution-status/08846e84-eae4-11eb-84be-00155d41f2fb + OUTPUT: { + "user": "admin", + "status": "finished", + "func_name": "update", + "created": "2021-07-22T14:42:56.284740Z", + "finished": "2021-07-22T14:43:01.813971Z", + "last_updated": "2021-07-22T14:42:56.284797Z", + "input_params": { + "uuid": "ee11541c-eaee-11eb-942c-00155d41f2fb", + "defaults": "{\"title\":\"pippo\"}" + }, + "output_params": { + "output": { + "uuid": "ee11541c-eaee-11eb-942c-00155d41f2fb" + } + } + } + """ + config = Configuration.load() + resource = get_object_or_404(ResourceBase, pk=pk) + if ( + config.read_only + or config.maintenance + or request.user.is_anonymous + or not request.user.is_authenticated + or resource is None + or not request.user.has_perm("view_resourcebase", resource.get_self_resource()) + ): + return Response(status=status.HTTP_403_FORBIDDEN) + if not resource.is_copyable: + return Response({"message": "Resource can not be cloned."}, status=400) + try: + request_params = self._get_request_params(request) + + _exec_request = ExecutionRequest.objects.create( + user=request.user, + func_name="copy", + geonode_resource=resource, + action="copy", + input_params={ + "instance": resource.id, + "owner": request_params.get("owner", request.user.username), + "defaults": request_params.get("defaults", "{}"), + }, + ) + resouce_service_dispatcher.apply_async(args=(str(_exec_request.exec_id),), expiration=30) + return Response( + { + "status": _exec_request.status, + "execution_id": _exec_request.exec_id, + "status_url": urljoin( + settings.SITEURL, reverse("rs-execution-status", kwargs={"execution_id": _exec_request.exec_id}) + ), + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + logger.exception(e) + return Response(status=status.HTTP_400_BAD_REQUEST, exception=e) + + @extend_schema( + methods=["put"], responses={200}, description="API endpoint allowing to set thumbnail of the Resource." + ) + @action( + detail=True, + url_path="set_thumbnail", + url_name="set_thumbnail", + methods=["put"], + permission_classes=[IsAuthenticated, UserHasPerms], + parser_classes=[JSONParser, MultiPartParser], + ) + def set_thumbnail(self, request, pk, *args, **kwargs): + resource = get_object_or_404(ResourceBase, pk=pk) + + if not request.data.get("file"): + raise ValidationError("Field file is required") + + file_data = request.data["file"] + + if isinstance(file_data, str): + if re.match(BASE64_PATTERN, file_data): + try: + thumbnail, _thumbnail_format = _decode_base64(file_data) + except Exception: + return Response( + "The request body is not a valid base64 string or the image format is not PNG or JPEG", + status=status.HTTP_400_BAD_REQUEST, + ) + else: + try: + # Check if file_data is a valid url and set it as thumbail_url + validate = URLValidator() + validate(file_data) + if urlparse(file_data).path.rsplit(".")[-1] not in ["png", "jpeg", "jpg"]: + return Response( + "The url must be of an image with format (png, jpeg or jpg)", + status=status.HTTP_400_BAD_REQUEST, + ) + resource.thumbnail_url = file_data + resource.save() + return Response({"thumbnail_url": resource.thumbnail_url}) + except Exception: + raise ValidationError( + detail="file is either a file upload, ASCII byte string or a valid image url string" + ) + else: + # Validate size + if file_data.size > 1000000: + raise ValidationError(detail="File must not exceed 1MB") + + thumbnail = file_data.read() + try: + file_data.seek(0) + Image.open(file_data) + except Exception: + raise ValidationError(detail="Invalid data provided") + if thumbnail: + resource_manager.set_thumbnail(resource.uuid, instance=resource, thumbnail=thumbnail) + return Response({"thumbnail_url": resource.thumbnail_url}) + return Response("Unable to set thumbnail", status=status.HTTP_400_BAD_REQUEST) + + @extend_schema( + methods=["get", "put", "delete", "post"], description="Get/Update/Delete/Add extra metadata for resource" + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + permission_classes=[IsOwnerOrAdmin, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], + url_path=r"extra_metadata", # noqa + url_name="extra-metadata", + ) + def extra_metadata(self, request, pk, *args, **kwargs): + _obj = get_object_or_404(ResourceBase, pk=pk) + + if request.method == "GET": + # get list of available metadata + queryset = _obj.metadata.all() + _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()] + if _filters: + queryset = queryset.filter(**_filters[0]) + return Response(ExtraMetadataSerializer().to_representation(queryset)) + if not request.method == "DELETE": + try: + extra_metadata = validate_extra_metadata(request.data, _obj) + except Exception as e: + return Response(status=500, data=e.args[0]) + + if request.method == "PUT": + """ + update specific metadata. The ID of the metadata is required to perform the update + [ + { + "id": 1, + "name": "foo_name", + "slug": "foo_sug", + "help_text": "object", + "field_type": "int", + "value": "object", + "category": "object" + } + ] + """ + for _m in extra_metadata: + _id = _m.pop("id") + ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) + logger.info("metadata updated for the selected resource") + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) + elif request.method == "DELETE": + # delete single metadata + """ + Expect a payload with the IDs of the metadata that should be deleted. Payload be like: + [4, 3] + """ + ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete() + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) + elif request.method == "POST": + # add new metadata + """ + [ + { + "name": "foo_name", + "slug": "foo_sug", + "help_text": "object", + "field_type": "int", + "value": "object", + "category": "object" + } + ] + """ + for _m in extra_metadata: + new_m = ExtraMetadata.objects.create(resource=_obj, metadata=_m) + new_m.save() + _obj.metadata.add(new_m) + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201) + + def _get_request_params(self, request, encode=False): + try: + return ( + QueryDict(request.body, mutable=True, encoding="UTF-8") + if encode + else QueryDict(request.body, mutable=True) + ) + except Exception as e: + """ + The request with the barer token access to the request.data during the token verification + so in this case if the request.body cannot not access, we just re-access to the + request.data to get the params needed + """ + logger.debug(e) + return request.data + + @extend_schema(methods=["get", "post", "delete"], description="Get Linked Resources") + @action( + detail=True, + methods=["get", "post", "delete"], + permission_classes=[UserHasPerms(perms_dict={"default": {"GET": ["base.view_resourcebase"]}})], + url_path=r"linked_resources", # noqa + url_name="linked_resources", + ) + def linked_resources(self, request, pk, *args, **kwargs): + resource = self.get_object() + if request.method in ("POST", "DELETE"): + success_var = [] + error_var = [] + payload = {"success": success_var, "error": error_var, "message": "Resources updated successfully"} + + target_ids = request.data.get("target") + if not isinstance(target_ids, list): + raise ValidationError("Payload is not valid") + + # remove duplicates and self ref + target_ids = set(target_ids) + if resource.id in target_ids: + error_var.append(resource.id) + + valid_ids = target_ids - {resource.id} + + for t_id in valid_ids: + try: + target = get_object_or_404(ResourceBase, pk=t_id) + + if request.method == "POST": + _, created = LinkedResource.objects.get_or_create(source=resource, target=target) + if created: + success_var.append(t_id) + continue + error_var.append(t_id) + if request.method == "DELETE": + link = LinkedResource.objects.filter(source=resource.id, target=t_id).first() + if not link: + logger.error(f"Resource selected with id {t_id} does not exist") + error_var.append(t_id) + continue + link.delete() + success_var.append(t_id) + except Exception: + error_var.append(t_id) + logger.error(f"Resource with id {t_id} not found") + + if len(error_var): + payload["message"] = "Some error has occurred during the saving" + return Response(payload, status=400) + + return Response(payload, status=200) + + return base_linked_resources(self.get_object().get_real_instance(), request.user, request.GET) + + +def base_linked_resources(instance, user, params): + try: + return Response(base_linked_resources_payload(instance, user, params)) + except Exception as e: + logger.exception(e) + return Response(data={"message": e.args[0], "success": False}, status=500, exception=True) + + +def base_linked_resources_payload(instance, user, params={}): + resource_type = params.get("resource_type", None) + link_type = params.get("link_type", None) + type_list = resource_type.split(",") if resource_type else [] + + warnings = {} + + if "page_size" in params or "page" in params: + warnings["PAGINATION"] = "Pagination is not supported on this call" + + ret = {"WARNINGS": warnings} + + get_visible_resources_p = functools.partial( + get_visible_resources, + user=user, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + if not link_type or link_type == "linked_to": + # list of linked resources, probably extended by ResourceBase's child class - may be loopable only once + linked_to_over = instance.get_linked_resources() + + # resolve the ids of linked resources - using either e QuerySet (preferred) or a list + if isinstance(linked_to_over, QuerySet): + linked_to_over_loopable = linked_to_over + linked_to_id_values = linked_to_over.values("target_id") + else: + linked_to_over_loopable = [lr for lr in linked_to_over] + linked_to_id_values = [lr.target_id for lr in linked_to_over_loopable] + + # filter resources by visibility / permissions + linked_to_visib = get_visible_resources_p(ResourceBase.objects.filter(id__in=linked_to_id_values)).order_by( + "-pk" + ) + # optionally filter by resource type + linked_to_visib = linked_to_visib.filter(resource_type__in=type_list) if type_list else linked_to_visib + linked_to_visib_ids = linked_to_visib.values_list("id", flat=True) + linked_to = [lres for lres in linked_to_over_loopable if lres.target.id in linked_to_visib_ids] + + ret["linked_to"] = LinkedResourceSerializer(linked_to, embed=True, many=True).data + + if not link_type or link_type == "linked_by": + linked_by_over = instance.get_linked_resources(as_target=True) + if isinstance(linked_by_over, QuerySet): + linked_by_over_loopable = linked_by_over + linked_by_id_values = linked_by_over.values("source_id") + else: + linked_by_over_loopable = [lr for lr in linked_by_over] + linked_by_id_values = [lr.source_id for lr in linked_by_over_loopable] + + linked_by_visib = get_visible_resources_p(ResourceBase.objects.filter(id__in=linked_by_id_values)).order_by( + "-pk" + ) + + linked_by_visib = linked_by_visib.filter(resource_type__in=type_list) if type_list else linked_by_visib + linked_by_visib_ids = linked_by_visib.values_list("id", flat=True) + linked_by = [lres for lres in linked_by_over_loopable if lres.source.id in linked_by_visib_ids] + + ret["linked_by"] = LinkedResourceSerializer( + instance=linked_by, serialize_source=True, embed=True, many=True + ).data + + if not ret["WARNINGS"]: + ret.pop("WARNINGS") + + return ret diff --git a/geonode/base/apps.py b/geonode/base/apps.py new file mode 100644 index 0000000..4b24947 --- /dev/null +++ b/geonode/base/apps.py @@ -0,0 +1,38 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig +from django.utils.translation import gettext_noop as _ + +from geonode.notifications_helper import NotificationsAppConfigBase + + +class BaseAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.base" + NOTIFICATIONS = ( + ( + "request_download_resourcebase", + _("Request to download a resource"), + _("A request for downloading a resource was sent"), + ), + ( + "request_resource_edit", + _("Request resource change"), + _("Owner has requested permissions to modify a resource"), + ), + ) diff --git a/geonode/base/auth.py b/geonode/base/auth.py new file mode 100644 index 0000000..e56d03e --- /dev/null +++ b/geonode/base/auth.py @@ -0,0 +1,273 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import re +import base64 +import datetime +import logging +import traceback +import ipaddress +from django.utils import timezone +from django.conf import settings +from django.contrib.auth import authenticate +from oauth2_provider.models import AccessToken, get_application_model +from oauthlib.common import generate_token +from django.contrib.auth.models import AnonymousUser + +logger = logging.getLogger(__name__) + + +def extract_user_from_headers(request): + user = AnonymousUser() + if "HTTP_AUTHORIZATION" in request.META: + auth_header = request.META.get("HTTP_AUTHORIZATION", request.META.get("HTTP_AUTHORIZATION2")) + + if auth_header and "Basic" in auth_header: + user = basic_auth_authenticate_user(auth_header) + elif auth_header and "Bearer" in auth_header: + user = token_header_authenticate_user(auth_header) + + if "apikey" in request.GET: + user = get_auth_user_from_token(request.GET.get("apikey")) + return user + + +def extract_headers(request): + """ + Extracts headers from the Django request object + :param request: The current django.http.HttpRequest object + :return: a dictionary with OAuthLib needed headers + """ + headers = request.META.copy() + if "wsgi.input" in headers: + del headers["wsgi.input"] + if "wsgi.errors" in headers: + del headers["wsgi.errors"] + if "HTTP_AUTHORIZATION" in headers: + headers["Authorization"] = headers["HTTP_AUTHORIZATION"] + + return headers + + +def make_token_expiration(seconds=86400): + _expire_seconds = getattr(settings, "ACCESS_TOKEN_EXPIRE_SECONDS", seconds) + _expire_time = datetime.datetime.now(timezone.get_current_timezone()) + _expire_delta = datetime.timedelta(seconds=_expire_seconds) + return _expire_time + _expire_delta + + +def create_auth_token(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): + if not user or user.is_anonymous: + return None + expires = make_token_expiration() + try: + Application = get_application_model() + app = Application.objects.get(name=client) + (access_token, created) = AccessToken.objects.get_or_create( + user=user, application=app, expires=expires, token=generate_token() + ) + return access_token + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + + +def extend_token(token): + try: + access_token = AccessToken.objects.get(id=token.id) + expires = make_token_expiration() + access_token.expires = expires + access_token.save() + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + + +def get_auth_token(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): + if not user or user.is_anonymous or not user.is_authenticated: + return None + try: + Application = get_application_model() + app = Application.objects.get(name=client) + access_token = AccessToken.objects.filter(user=user, application=app).order_by("-expires").first() + return access_token + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + return None + + +def get_auth_user(access_token, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): + try: + Application = get_application_model() + app = Application.objects.get(name=client) + user = AccessToken.objects.filter(token=access_token, application=app).order_by("-expires").first().user + return user + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + return None + + +def get_or_create_token(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): + if not user or user.is_anonymous: + return None + try: + application = get_application_model() + app = application.objects.get(name=client) + + # Let's create the new AUTH TOKEN + existing_token = None + try: + existing_token = AccessToken.objects.filter(user=user, application=app).order_by("-expires").first() + if existing_token and existing_token.is_expired(): + existing_token.delete() + existing_token = None + except Exception: + existing_token = None + tb = traceback.format_exc() + if tb: + logger.debug(tb) + + if not existing_token: + token = create_auth_token(user, client) + else: + token = existing_token + + return token + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + + +def delete_old_tokens(user, client=settings.OAUTH2_DEFAULT_BACKEND_CLIENT_NAME): + if not user or user.is_anonymous: + return None + try: + application = get_application_model() + app = application.objects.get(name=client) + + # Lets delete the old one + old_tokens = AccessToken.objects.filter(user=user, application=app).order_by("-expires") + for old in old_tokens: + if old.is_expired(): + old.delete() + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + + +def get_token_from_auth_header(auth_header, create_if_not_exists=False): + if re.search("Basic", auth_header, re.IGNORECASE): + user = basic_auth_authenticate_user(auth_header) + if user and user.is_active: + return get_auth_token(user) if not create_if_not_exists else get_or_create_token(user) + elif re.search("Bearer", auth_header, re.IGNORECASE): + return re.compile(re.escape("Bearer "), re.IGNORECASE).sub("", auth_header) + return None + + +def set_session_token(session, token): + session["access_token"] = str(token) + + +def get_session_token(session): + return session.get("access_token", None) + + +def get_token_object_from_session(session): + if "access_token" in session: + try: + return AccessToken.objects.get(token=get_session_token(session)) + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + del session["access_token"] + session.modified = True + return None + return None + + +def remove_session_token(session): + if "access_token" in session: + del session["access_token"] + + +def basic_auth_authenticate_user(auth_header: str): + """ + Function performing user authentication based on BasicAuth Authorization header + + :param auth_header: Authorization header of the request + """ + encoded_credentials = auth_header.split(" ")[1] # Removes "Basic " to isolate credentials + decoded_credentials = base64.b64decode(encoded_credentials).decode("utf-8").split(":") + username = decoded_credentials[0] + password = decoded_credentials[1] + + return authenticate(username=username, password=password) + + +def get_token_object(token): + try: + access_token = AccessToken.objects.filter(token=token).first() + if access_token and access_token.is_valid(): + return access_token + except Exception: + tb = traceback.format_exc() + if tb: + logger.debug(tb) + return None + + +def get_auth_user_from_token(token): + access_token = get_token_object(token) + if access_token: + return access_token.user + + +def token_header_authenticate_user(auth_header: str): + token = get_token_from_auth_header(auth_header) + return get_auth_user_from_token(token) + + +def visitor_ip_address(request): + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") + + if x_forwarded_for: + ip = x_forwarded_for.split(",")[0] + else: + ip = request.META.get("REMOTE_ADDR") + if ip: + ip = re.match(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})", ip)[0] + return ip + + +def is_ipaddress_in_whitelist(visitor_ip, whitelist): + # Chech if an IP is in the whitelisted IP ranges + _results = [] + if visitor_ip and whitelist and len(whitelist) > 0: + visitor_ipaddress = ipaddress.ip_address(visitor_ip) + _results = [visitor_ipaddress in ipaddress.ip_network(wip) for wip in whitelist] + return False if not _results else any(_results) diff --git a/geonode/base/base_urls.py b/geonode/base/base_urls.py new file mode 100644 index 0000000..8d97519 --- /dev/null +++ b/geonode/base/base_urls.py @@ -0,0 +1,46 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.urls import re_path + +from . import views + +js_info_dict = { + "packages": ("geonode.base",), +} + +urlpatterns = [ + # 'geonode.resourcebases.views', + re_path(r"^(?P\d+)/metadata$", views.resourcebase_metadata, name="resourcebase_metadata"), + re_path( + r"^(?P[^/]*)/metadata_detail$", + views.resourcebase_metadata_detail, + name="resourcebase_metadata_detail", + ), + re_path( + r"^(?P\d+)/metadata_advanced$", + views.resourcebase_metadata_advanced, + name="resourcebase_metadata_advanced", + ), + re_path( + r"^(?P[^/]+)/embed$", + views.resourcebase_embed, + {"template": "base/base_embed.html"}, + name="resourcebase_embed", + ), +] diff --git a/geonode/base/bbox_utils.py b/geonode/base/bbox_utils.py new file mode 100644 index 0000000..b589e1b --- /dev/null +++ b/geonode/base/bbox_utils.py @@ -0,0 +1,328 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import math +import copy +import json +import re +import logging + +from decimal import Decimal +from typing import Union, List, Generator + +from pyproj import CRS +from shapely import affinity +from shapely.ops import split +from shapely.geometry import mapping, Polygon, LineString, GeometryCollection + +from django.contrib.gis.geos import Polygon as DjangoPolygon + +from geonode import GeoNodeException +from geonode.utils import bbox_to_projection + +logger = logging.getLogger(__name__) + + +class BBOXHelper: + """ + A bounding box representation to avoid use of list indices when + dealing with bounding boxes. + """ + + def __init__(self, minmaxform): + self.xmin, self.ymin, self.xmax, self.ymax = minmaxform + + @classmethod + def from_xy(cls, xy): + """ + Constructs a `BBOXHelper` instance from coordinates in "xy" format. + + :param xy: collection of coordinates as [xmin, xmax, ymin, ymax] + """ + xy[1], xy[2] = xy[2], xy[1] + return cls(xy) + + def as_polygon(self): + return DjangoPolygon.from_bbox((self.xmin, self.ymin, self.xmax, self.ymax)) + + +def normalize_x_value(value): + """ + Normalise x-axis value/longtitude to fall within [-180, 180] + """ + return ((value + 180) % 360) - 180 + + +def polygon_from_bbox(bbox, srid=4326): + """ + Constructs a Polygon object with srid from a provided bbox. + """ + poly = DjangoPolygon.from_bbox(bbox) + poly.srid = srid + return poly + + +def filter_bbox(queryset, bbox): + """ + Filters a queryset by a provided bounding box. + + :param bbox: Comma-separated coordinates as "xmin,ymin,xmax,ymax" + """ + assert queryset.model.__class__.__name__ == "PolymorphicModelBase" + + bboxes = [] + _bbox_index = -1 + for _x, _y in enumerate(bbox.split(",")): + if _x % 4 == 0: + bboxes.append([]) + _bbox_index += 1 + bboxes[_bbox_index].append(_y) + + search_queryset = None + for _bbox in bboxes: + _bbox = list(map(Decimal, _bbox)) + search_polygon = polygon_from_bbox((_bbox[0], _bbox[1], _bbox[2], _bbox[3])) + for search_polygon_dl in [ + DjangoPolygon.from_ewkt(_p.wkt) + for _p in split_polygon(json.loads(search_polygon.json), output_format="polygons") + ]: + _qs = queryset.filter(ll_bbox_polygon__intersects=search_polygon_dl) + search_queryset = _qs if search_queryset is None else search_queryset | _qs + + return search_queryset + + +def check_crossing(lon1: float, lon2: float, validate: bool = False, dlon_threshold: float = 180.0): + """ + ref.: https://towardsdatascience.com/around-the-world-in-80-lines-crossing-the-antimeridian-with-python-and-shapely-c87c9b6e1513 + Assuming a minimum travel distance between two provided longitude coordinates, + checks if the 180th meridian (antimeridian) is crossed. + """ + + if validate and any([abs(x) > 180.0 for x in [lon1, lon2]]): + raise ValueError("longitudes must be in degrees [-180.0, 180.0]") + return abs(lon2 - lon1) > dlon_threshold + + +def translate_polygons( + geometry_collection: GeometryCollection, output_format: str = "geojson" +) -> Generator[Union[List[dict], List[Polygon]], None, None]: + """ + ref.: https://towardsdatascience.com/around-the-world-in-80-lines-crossing-the-antimeridian-with-python-and-shapely-c87c9b6e1513 + """ + for polygon in geometry_collection: + (minx, _, maxx, _) = polygon.bounds + if minx < -180: + geo_polygon = affinity.translate(polygon, xoff=360) + elif maxx > 180: + geo_polygon = affinity.translate(polygon, xoff=-360) + else: + geo_polygon = polygon + + yield_geojson = output_format == "geojson" + yield json.dumps(mapping(geo_polygon)) if (yield_geojson) else geo_polygon + + +def split_polygon( + geojson: dict, output_format: str = "geojson", validate: bool = False +) -> Union[List[dict], List[Polygon], GeometryCollection]: + """ + ref.: https://towardsdatascience.com/around-the-world-in-80-lines-crossing-the-antimeridian-with-python-and-shapely-c87c9b6e1513 + Given a GeoJSON representation of a Polygon, returns a collection of + 'antimeridian-safe' constituent polygons split at the 180th meridian, + ensuring compliance with GeoJSON standards (https://tools.ietf.org/html/rfc7946#section-3.1.9) + Assumptions: + - Any two consecutive points with over 180 degrees difference in + longitude are assumed to cross the antimeridian + - The polygon spans less than 360 degrees in longitude (i.e. does not wrap around the globe) + - However, the polygon may cross the antimeridian on multiple occasions + Parameters: + geojson (dict): GeoJSON of input polygon to be split. For example: + { + "type": "Polygon", + "coordinates": [ + [ + [179.0, 0.0], [-179.0, 0.0], [-179.0, 1.0], + [179.0, 1.0], [179.0, 0.0] + ] + ] + } + output_format (str): Available options: "geojson", "polygons", "geometrycollection" + If "geometrycollection" returns a Shapely GeometryCollection. + Otherwise, returns a list of either GeoJSONs or Shapely Polygons + validate (bool): Checks if all longitudes are within [-180.0, 180.0] + + Returns: + List[dict]/List[Polygon]/GeometryCollection: antimeridian-safe polygon(s) + """ + output_format = output_format.replace("-", "").strip().lower() + coords_shift = copy.deepcopy(geojson["coordinates"]) + shell_minx = shell_maxx = None + split_meridians = set() + splitter = None + + for ring_index, ring in enumerate(coords_shift): + if len(ring) < 1: + continue + else: + ring_minx = ring_maxx = ring[0][0] + crossings = 0 + + for coord_index, (lon, _) in enumerate(ring[1:], start=1): + lon_prev = ring[coord_index - 1][0] # [0] corresponds to longitude coordinate + if check_crossing(lon, lon_prev, validate=validate): + direction = math.copysign(1, lon - lon_prev) + coords_shift[ring_index][coord_index][0] = lon - (direction * 360.0) + crossings += 1 + + x_shift = coords_shift[ring_index][coord_index][0] + if x_shift < ring_minx: + ring_minx = x_shift + if x_shift > ring_maxx: + ring_maxx = x_shift + + # Ensure that any holes remain contained within the (translated) outer shell + if ring_index == 0: # by GeoJSON definition, first ring is the outer shell + shell_minx, shell_maxx = (ring_minx, ring_maxx) + elif ring_minx < shell_minx: + ring_shift = [[x + 360, y] for (x, y) in coords_shift[ring_index]] + coords_shift[ring_index] = ring_shift + ring_minx, ring_maxx = (x + 360 for x in (ring_minx, ring_maxx)) + elif ring_maxx > shell_maxx: + ring_shift = [[x - 360, y] for (x, y) in coords_shift[ring_index]] + coords_shift[ring_index] = ring_shift + ring_minx, ring_maxx = (x - 360 for x in (ring_minx, ring_maxx)) + + if crossings: # keep track of meridians to split on + if ring_minx < -180: + split_meridians.add(-180) + if ring_maxx > 180: + split_meridians.add(180) + + n_splits = len(split_meridians) + if n_splits > 1: + raise NotImplementedError( + """Splitting a Polygon by multiple meridians (MultiLineString) + not supported by Shapely""" + ) + elif n_splits == 1: + split_lon = next(iter(split_meridians)) + meridian = [[split_lon, -90.0], [split_lon, 90.0]] + splitter = LineString(meridian) + + shell, *holes = coords_shift if splitter else geojson["coordinates"] + if splitter: + split_polygons = split(Polygon(shell, holes), splitter) + else: + split_polygons = GeometryCollection([Polygon(shell, holes)]) + + geo_polygons = list(translate_polygons(split_polygons, output_format)) + if output_format == "geometrycollection": + return GeometryCollection(geo_polygons) + else: + return geo_polygons + + +def transform_bbox(bbox: List, target_crs: str = "EPSG:3857"): + """ + Function transforming BBOX in dataset compliant format (xmin, xmax, ymin, ymax, 'EPSG:xxxx') to another CRS, + preserving overflow values. + """ + match = re.match(r"^(EPSG:)?(?P\d{4,6})$", str(target_crs)) + target_srid = int(match.group("srid")) if match else 4326 + return list(bbox_to_projection(bbox, target_srid=target_srid))[:-1] + [target_crs] + + +def epsg_3857_area_of_use(target_crs="EPSG:4326"): + """ + Shortcut function, returning area of use of EPSG:3857 (in EPSG:4326) in a dataset compliant BBOX + """ + epsg3857 = CRS.from_user_input("EPSG:3857") + area_of_use = [ + getattr(epsg3857.area_of_use, "west"), + getattr(epsg3857.area_of_use, "east"), + getattr(epsg3857.area_of_use, "south"), + getattr(epsg3857.area_of_use, "north"), + "EPSG:4326", + ] + if target_crs != "EPSG:4326": + return transform_bbox(area_of_use, target_crs) + return area_of_use + + +def crop_to_3857_area_of_use(bbox: List) -> List: + # perform the comparison in EPSG:4326 (the pivot for EPSG:3857) + bbox4326 = transform_bbox(bbox, target_crs="EPSG:4326") + + # get area of use of EPSG:3857 in EPSG:4326 + epsg3857_bounds_bbox = epsg_3857_area_of_use() + + bbox = [] + for coord, bound_coord in zip(bbox4326[:-1], epsg3857_bounds_bbox[:-1]): + if abs(coord) > abs(bound_coord): + logger.debug("Thumbnail generation: cropping BBOX's coord to EPSG:3857 area of use.") + bbox.append(bound_coord) + else: + bbox.append(coord) + + bbox.append("EPSG:4326") + + return bbox + + +def exceeds_epsg3857_area_of_use(bbox: List) -> bool: + """ + Function checking if a provided BBOX extends the are of use of EPSG:3857. Comparison is performed after casting + the BBOX to EPSG:4326 (pivot for EPSG:3857). + + :param bbox: a dataset compliant BBOX in a certain CRS, in (xmin, xmax, ymin, ymax, 'EPSG:xxxx') order + :returns: List of indicators whether BBOX's coord exceeds the area of use of EPSG:3857 + """ + + # perform the comparison in EPSG:4326 (the pivot for EPSG:3857) + bbox4326 = transform_bbox(bbox, target_crs="EPSG:4326") + + # get area of use of EPSG:3857 in EPSG:4326 + epsg3857_bounds_bbox = epsg_3857_area_of_use() + + exceeds = False + for coord, bound_coord in zip(bbox4326[:-1], epsg3857_bounds_bbox[:-1]): + if abs(coord) > abs(bound_coord): + exceeds = True + + return exceeds + + +def clean_bbox(bbox, target_crs): + # make sure BBOX is provided with the CRS in a correct format + source_crs = bbox[-1] + + srid_regex = re.match(r"EPSG:\d+", source_crs) + if not srid_regex: + logger.error(f"Thumbnail bbox is in a wrong format: {bbox}") + raise GeoNodeException("Wrong BBOX format") + + # for the EPSG:3857 (default thumb's CRS) - make sure received BBOX can be transformed to the target CRS; + # if it can't be (original coords are outside of the area of use of EPSG:3857), thumbnail generation with + # the provided bbox is impossible. + if target_crs == "EPSG:3857" and bbox[-1].upper() != "EPSG:3857": + bbox = crop_to_3857_area_of_use(bbox) + + bbox = transform_bbox(bbox, target_crs=target_crs) + return bbox diff --git a/geonode/base/enumerations.py b/geonode/base/enumerations.py new file mode 100644 index 0000000..66430b4 --- /dev/null +++ b/geonode/base/enumerations.py @@ -0,0 +1,501 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.utils.translation import gettext_lazy as _ + +LINK_TYPES = ["original", "uploaded", "data", "image", "metadata", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS"] + +HIERARCHY_LEVELS = ( + ("series", _("series")), + ("software", _("computer program or routine")), + ("featureType", _("feature type")), + ("model", _("copy or imitation of an existing or hypothetical object")), + ("collectionHardware", _("collection hardware")), + ("collectionSession", _("collection session")), + ("nonGeographicDataset", _("non-geographic data")), + ("propertyType", _("property type")), + ("fieldSession", _("field session")), + ("dataset", _("dataset")), + ("service", _("service interfaces")), + ("attribute", _("attribute class")), + ("attributeType", _("characteristic of a feature")), + ("tile", _("tile or spatial subset of geographic data")), + ("feature", _("feature")), + ("dimensionGroup", _("dimension group")), +) + +UPDATE_FREQUENCIES = ( + ("unknown", _("frequency of maintenance for the data is not known")), + ("continual", _("data is repeatedly and frequently updated")), + ("notPlanned", _("there are no plans to update the data")), + ("daily", _("data is updated each day")), + ("annually", _("data is updated every year")), + ("asNeeded", _("data is updated as deemed necessary")), + ("monthly", _("data is updated each month")), + ("fortnightly", _("data is updated every two weeks")), + ("irregular", _("data is updated in intervals that are uneven in duration")), + ("weekly", _("data is updated on a weekly basis")), + ("biannually", _("data is updated twice each year")), + ("quarterly", _("data is updated every three months")), +) + +CONTACT_FIELDS = [ + "name", + "organization", + "position", + "voice", + "facsimile", + "delivery_point", + "city", + "administrative_area", + "postal_code", + "country", + "email", + "role", +] + +DEFAULT_SUPPLEMENTAL_INFORMATION = _(_("No information provided")) + +COUNTRIES = ( + ("AFG", "Afghanistan"), + ("ALA", "Aland Islands"), + ("ALB", "Albania"), + ("DZA", "Algeria"), + ("ASM", "American Samoa"), + ("AND", "Andorra"), + ("AGO", "Angola"), + ("AIA", "Anguilla"), + ("ATG", "Antigua and Barbuda"), + ("ARG", "Argentina"), + ("ARM", "Armenia"), + ("ABW", "Aruba"), + ("AUS", "Australia"), + ("AUT", "Austria"), + ("AZE", "Azerbaijan"), + ("BHS", "Bahamas"), + ("BHR", "Bahrain"), + ("BGD", "Bangladesh"), + ("BRB", "Barbados"), + ("BLR", "Belarus"), + ("BEL", "Belgium"), + ("BLZ", "Belize"), + ("BEN", "Benin"), + ("BMU", "Bermuda"), + ("BTN", "Bhutan"), + ("BOL", "Bolivia"), + ("BIH", "Bosnia and Herzegovina"), + ("BWA", "Botswana"), + ("BRA", "Brazil"), + ("VGB", "British Virgin Islands"), + ("BRN", "Brunei Darussalam"), + ("BGR", "Bulgaria"), + ("BFA", "Burkina Faso"), + ("BDI", "Burundi"), + ("KHM", "Cambodia"), + ("CMR", "Cameroon"), + ("CAN", "Canada"), + ("CPV", "Cape Verde"), + ("CYM", "Cayman Islands"), + ("CAF", "Central African Republic"), + ("TCD", "Chad"), + ("CIL", "Channel Islands"), + ("CHL", "Chile"), + ("CHN", "China"), + ("HKG", "China - Hong Kong"), + ("MAC", "China - Macao"), + ("COL", "Colombia"), + ("COM", "Comoros"), + ("COG", "Congo"), + ("COK", "Cook Islands"), + ("CRI", "Costa Rica"), + ("CIV", "Cote d'Ivoire"), + ("HRV", "Croatia"), + ("CUB", "Cuba"), + ("CYP", "Cyprus"), + ("CZE", "Czech Republic"), + ("PRK", "Democratic People's Republic of Korea"), + ("COD", "Democratic Republic of the Congo"), + ("DNK", "Denmark"), + ("DJI", "Djibouti"), + ("DMA", "Dominica"), + ("DOM", "Dominican Republic"), + ("ECU", "Ecuador"), + ("EGY", "Egypt"), + ("SLV", "El Salvador"), + ("GNQ", "Equatorial Guinea"), + ("ERI", "Eritrea"), + ("EST", "Estonia"), + ("ETH", "Ethiopia"), + ("FRO", "Faeroe Islands"), + ("FLK", "Falkland Islands (Malvinas)"), + ("FJI", "Fiji"), + ("FIN", "Finland"), + ("FRA", "France"), + ("GUF", "French Guiana"), + ("PYF", "French Polynesia"), + ("GAB", "Gabon"), + ("GMB", "Gambia"), + ("GEO", "Georgia"), + ("DEU", "Germany"), + ("GHA", "Ghana"), + ("GIB", "Gibraltar"), + ("GRC", "Greece"), + ("GRL", "Greenland"), + ("GRD", "Grenada"), + ("GLP", "Guadeloupe"), + ("GUM", "Guam"), + ("GTM", "Guatemala"), + ("GGY", "Guernsey"), + ("GIN", "Guinea"), + ("GNB", "Guinea-Bissau"), + ("GUY", "Guyana"), + ("HTI", "Haiti"), + ("VAT", "Holy See (Vatican City)"), + ("HND", "Honduras"), + ("HUN", "Hungary"), + ("ISL", "Iceland"), + ("IND", "India"), + ("IDN", "Indonesia"), + ("IRN", "Iran"), + ("IRQ", "Iraq"), + ("IRL", "Ireland"), + ("IMN", "Isle of Man"), + ("ISR", "Israel"), + ("ITA", "Italy"), + ("JAM", "Jamaica"), + ("JPN", "Japan"), + ("JEY", "Jersey"), + ("JOR", "Jordan"), + ("KAZ", "Kazakhstan"), + ("KEN", "Kenya"), + ("KIR", "Kiribati"), + ("KWT", "Kuwait"), + ("KGZ", "Kyrgyzstan"), + ("LAO", "Lao People's Democratic Republic"), + ("LVA", "Latvia"), + ("LBN", "Lebanon"), + ("LSO", "Lesotho"), + ("LBR", "Liberia"), + ("LBY", "Libyan Arab Jamahiriya"), + ("LIE", "Liechtenstein"), + ("LTU", "Lithuania"), + ("LUX", "Luxembourg"), + ("MKD", "Macedonia"), + ("MDG", "Madagascar"), + ("MWI", "Malawi"), + ("MYS", "Malaysia"), + ("MDV", "Maldives"), + ("MLI", "Mali"), + ("MLT", "Malta"), + ("MHL", "Marshall Islands"), + ("MTQ", "Martinique"), + ("MRT", "Mauritania"), + ("MUS", "Mauritius"), + ("MYT", "Mayotte"), + ("MEX", "Mexico"), + ("FSM", "Micronesia, Federated States of"), + ("MCO", "Monaco"), + ("MNG", "Mongolia"), + ("MNE", "Montenegro"), + ("MSR", "Montserrat"), + ("MAR", "Morocco"), + ("MOZ", "Mozambique"), + ("MMR", "Myanmar"), + ("NAM", "Namibia"), + ("NRU", "Nauru"), + ("NPL", "Nepal"), + ("NLD", "Netherlands"), + ("ANT", "Netherlands Antilles"), + ("NCL", "New Caledonia"), + ("NZL", "New Zealand"), + ("NIC", "Nicaragua"), + ("NER", "Niger"), + ("NGA", "Nigeria"), + ("NIU", "Niue"), + ("NFK", "Norfolk Island"), + ("MNP", "Northern Mariana Islands"), + ("NOR", "Norway"), + ("PSE", "Occupied Palestinian Territory"), + ("OMN", "Oman"), + ("PAK", "Pakistan"), + ("PLW", "Palau"), + ("PAN", "Panama"), + ("PNG", "Papua New Guinea"), + ("PRY", "Paraguay"), + ("PER", "Peru"), + ("PHL", "Philippines"), + ("PCN", "Pitcairn"), + ("POL", "Poland"), + ("PRT", "Portugal"), + ("PRI", "Puerto Rico"), + ("QAT", "Qatar"), + ("KOR", "Republic of Korea"), + ("MDA", "Republic of Moldova"), + ("REU", "Reunion"), + ("ROU", "Romania"), + ("RUS", "Russian Federation"), + ("RWA", "Rwanda"), + ("BLM", "Saint-Barthelemy"), + ("SHN", "Saint Helena"), + ("KNA", "Saint Kitts and Nevis"), + ("LCA", "Saint Lucia"), + ("MAF", "Saint-Martin (French part)"), + ("SPM", "Saint Pierre and Miquelon"), + ("VCT", "Saint Vincent and the Grenadines"), + ("WSM", "Samoa"), + ("SMR", "San Marino"), + ("STP", "Sao Tome and Principe"), + ("SAU", "Saudi Arabia"), + ("SEN", "Senegal"), + ("SRB", "Serbia"), + ("SYC", "Seychelles"), + ("SLE", "Sierra Leone"), + ("SGP", "Singapore"), + ("SVK", "Slovakia"), + ("SVN", "Slovenia"), + ("SLB", "Solomon Islands"), + ("SOM", "Somalia"), + ("ZAF", "South Africa"), + ("SSD", "South Sudan"), + ("ESP", "Spain"), + ("LKA", "Sri Lanka"), + ("SDN", "Sudan"), + ("SUR", "Suriname"), + ("SJM", "Svalbard and Jan Mayen Islands"), + ("SWZ", "Swaziland"), + ("SWE", "Sweden"), + ("CHE", "Switzerland"), + ("SYR", "Syrian Arab Republic"), + ("TJK", "Tajikistan"), + ("THA", "Thailand"), + ("TLS", "Timor-Leste"), + ("TGO", "Togo"), + ("TKL", "Tokelau"), + ("TON", "Tonga"), + ("TTO", "Trinidad and Tobago"), + ("TUN", "Tunisia"), + ("TUR", "Turkey"), + ("TKM", "Turkmenistan"), + ("TCA", "Turks and Caicos Islands"), + ("TUV", "Tuvalu"), + ("UGA", "Uganda"), + ("UKR", "Ukraine"), + ("ARE", "United Arab Emirates"), + ("GBR", "United Kingdom"), + ("TZA", "United Republic of Tanzania"), + ("USA", "United States of America"), + ("VIR", "United States Virgin Islands"), + ("URY", "Uruguay"), + ("UZB", "Uzbekistan"), + ("VUT", "Vanuatu"), + ("VEN", "Venezuela (Bolivarian Republic of)"), + ("VNM", "Viet Nam"), + ("WLF", "Wallis and Futuna Islands"), + ("ESH", "Western Sahara"), + ("YEM", "Yemen"), + ("ZMB", "Zambia"), + ("ZWE", "Zimbabwe"), +) + +# Taken from http://www.w3.org/WAI/ER/IG/ert/iso639.htm +ALL_LANGUAGES = ( + ("abk", "Abkhazian"), + ("aar", "Afar"), + ("afr", "Afrikaans"), + ("amh", "Amharic"), + ("ara", "Arabic"), + ("asm", "Assamese"), + ("aym", "Aymara"), + ("aze", "Azerbaijani"), + ("bak", "Bashkir"), + ("ben", "Bengali"), + ("bih", "Bihari"), + ("bis", "Bislama"), + ("bre", "Breton"), + ("bul", "Bulgarian"), + ("bel", "Byelorussian"), + ("cat", "Catalan"), + ("chi", "Chinese"), + ("cos", "Corsican"), + ("dan", "Danish"), + ("dzo", "Dzongkha"), + ("eng", "English"), + ("fra", "French"), + ("epo", "Esperanto"), + ("est", "Estonian"), + ("fao", "Faroese"), + ("fij", "Fijian"), + ("fin", "Finnish"), + ("fry", "Frisian"), + ("glg", "Gallegan"), + ("ger", "German"), + ("gre", "Greek"), + ("kal", "Greenlandic"), + ("grn", "Guarani"), + ("guj", "Gujarati"), + ("hau", "Hausa"), + ("heb", "Hebrew"), + ("hin", "Hindi"), + ("hun", "Hungarian"), + ("ind", "Indonesian"), + ("ina", "Interlingua (International Auxiliary language Association)"), + ("iku", "Inuktitut"), + ("ipk", "Inupiak"), + ("ita", "Italian"), + ("jpn", "Japanese"), + ("kan", "Kannada"), + ("kas", "Kashmiri"), + ("kaz", "Kazakh"), + ("khm", "Khmer"), + ("kin", "Kinyarwanda"), + ("kir", "Kirghiz"), + ("kor", "Korean"), + ("kur", "Kurdish"), + ("oci", "Langue d 'Oc (post 1500)"), + ("lao", "Lao"), + ("lat", "Latin"), + ("lav", "Latvian"), + ("lin", "Lingala"), + ("lit", "Lithuanian"), + ("mlg", "Malagasy"), + ("mlt", "Maltese"), + ("mar", "Marathi"), + ("mol", "Moldavian"), + ("mon", "Mongolian"), + ("nau", "Nauru"), + ("nep", "Nepali"), + ("nor", "Norwegian"), + ("ori", "Oriya"), + ("orm", "Oromo"), + ("pan", "Panjabi"), + ("pol", "Polish"), + ("por", "Portuguese"), + ("pus", "Pushto"), + ("que", "Quechua"), + ("roh", "Rhaeto-Romance"), + ("run", "Rundi"), + ("rus", "Russian"), + ("smo", "Samoan"), + ("sag", "Sango"), + ("san", "Sanskrit"), + ("scr", "Serbo-Croatian"), + ("sna", "Shona"), + ("snd", "Sindhi"), + ("sin", "Singhalese"), + ("ssw", "Siswant"), + ("slv", "Slovenian"), + ("som", "Somali"), + ("sot", "Sotho"), + ("spa", "Spanish"), + ("sun", "Sudanese"), + ("swa", "Swahili"), + ("tgl", "Tagalog"), + ("tgk", "Tajik"), + ("tam", "Tamil"), + ("tat", "Tatar"), + ("tel", "Telugu"), + ("tha", "Thai"), + ("tir", "Tigrinya"), + ("tog", "Tonga (Nyasa)"), + ("tso", "Tsonga"), + ("tsn", "Tswana"), + ("tur", "Turkish"), + ("tuk", "Turkmen"), + ("twi", "Twi"), + ("uig", "Uighur"), + ("ukr", "Ukrainian"), + ("urd", "Urdu"), + ("uzb", "Uzbek"), + ("vie", "Vietnamese"), + ("vol", "Volapük"), + ("wol", "Wolof"), + ("xho", "Xhosa"), + ("yid", "Yiddish"), + ("yor", "Yoruba"), + ("zha", "Zhuang"), + ("zul", "Zulu"), +) + +CHARSETS = ( + ("", "None/Unknown"), + ("UTF-8", "UTF-8/Unicode"), + ("ISO-8859-1", "Latin1/ISO-8859-1"), + ("ISO-8859-2", "Latin2/ISO-8859-2"), + ("ISO-8859-3", "Latin3/ISO-8859-3"), + ("ISO-8859-4", "Latin4/ISO-8859-4"), + ("ISO-8859-5", "Latin5/ISO-8859-5"), + ("ISO-8859-6", "Latin6/ISO-8859-6"), + ("ISO-8859-7", "Latin7/ISO-8859-7"), + ("ISO-8859-8", "Latin8/ISO-8859-8"), + ("ISO-8859-9", "Latin9/ISO-8859-9"), + ("ISO-8859-10", "Latin10/ISO-8859-10"), + ("ISO-8859-13", "Latin13/ISO-8859-13"), + ("ISO-8859-14", "Latin14/ISO-8859-14"), + ("ISO8859-15", "Latin15/ISO-8859-15"), + ("Big5", "BIG5"), + ("EUC-JP", "EUC-JP"), + ("EUC-KR", "EUC-KR"), + ("GBK", "GBK"), + ("GB18030", "GB18030"), + ("Shift_JIS", "Shift_JIS"), + ("KOI8-R", "KOI8-R"), + ("KOI8-U", "KOI8-U"), + ("cp874", "Windows CP874"), + ("windows-1250", "Windows CP1250"), + ("windows-1251", "Windows CP1251"), + ("windows-1252", "Windows CP1252"), + ("windows-1253", "Windows CP1253"), + ("windows-1254", "Windows CP1254"), + ("windows-1255", "Windows CP1255"), + ("windows-1256", "Windows CP1256"), + ("windows-1257", "Windows CP1257"), + ("windows-1258", "Windows CP1258"), +) + +STATE_READY = "READY" +STATE_RUNNING = "RUNNING" +STATE_PENDING = "PENDING" +STATE_WAITING = "WAITING" +STATE_INCOMPLETE = "INCOMPLETE" +STATE_COMPLETE = "COMPLETE" +STATE_INVALID = "INVALID" +STATE_PROCESSED = "PROCESSED" + +PROCESSING_STATES = ( + (STATE_READY, "READY"), + (STATE_RUNNING, "RUNNING"), + (STATE_PENDING, "PENDING"), + (STATE_WAITING, "WAITING"), + (STATE_INCOMPLETE, "INCOMPLETE"), + (STATE_COMPLETE, "COMPLETE"), + (STATE_INVALID, "INVALID"), + (STATE_PROCESSED, "PROCESSED"), +) + +SOURCE_TYPE_LOCAL = "LOCAL" +SOURCE_TYPE_REMOTE = "REMOTE" +SOURCE_TYPE_COPYREMOTE = "COPYREMOTE" + +SOURCE_TYPES = ( + (SOURCE_TYPE_LOCAL, "LOCAL"), + (SOURCE_TYPE_REMOTE, "REMOTE"), + (SOURCE_TYPE_COPYREMOTE, "COPYREMOTE"), +) + +LAYER_TYPES = ["vector", "raster", "remote", "vector_time"] diff --git a/geonode/base/fields.py b/geonode/base/fields.py new file mode 100644 index 0000000..b1e5f0b --- /dev/null +++ b/geonode/base/fields.py @@ -0,0 +1,28 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django import forms + + +class MultiThesauriField(forms.ModelMultipleChoiceField): + def label_from_instance(self, obj): + # Note: Not using .get() because filter()[0] is used in original + # code. The hard-coded language is currently used throughout + # geonode. + return obj.keyword.first().label diff --git a/geonode/base/fixtures/bobby.json b/geonode/base/fixtures/bobby.json new file mode 100644 index 0000000..f9ea8ac --- /dev/null +++ b/geonode/base/fixtures/bobby.json @@ -0,0 +1,20 @@ +[ +{ + "pk": 1, + "model": "people.Profile", + "fields": { + "username": "bobby", + "first_name": "bobby", + "last_name": "", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2010-06-10 16:59:13", + "groups": [], + "user_permissions": [], + "password": "sha1$8d019$a84eea3f5093eed93bc68bf62fe400f14042ab06", + "email": "bobby@bob.com", + "date_joined": "2010-06-10 16:58:18" + } +} +] diff --git a/geonode/base/fixtures/default_oauth_apps.json b/geonode/base/fixtures/default_oauth_apps.json new file mode 100644 index 0000000..41b8bd9 --- /dev/null +++ b/geonode/base/fixtures/default_oauth_apps.json @@ -0,0 +1,21 @@ +[ +{ + "model": "oauth2_provider.application", + "pk": 1001, + "fields": { + "skip_authorization": true, + "created": "2018-05-31T10:00:31.661Z", + "updated": "2018-05-31T11:30:31.245Z", + "algorithm": "RS256", + "redirect_uris": "http://localhost:8080/geoserver/index.html\nhttp://localhost/geoserver/index.html", + "name": "GeoServer", + "authorization_grant_type": "authorization-code", + "client_type": "confidential", + "client_id": "Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV", + "client_secret": "rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3", + "user": [ + "admin" + ] + } +} +] diff --git a/geonode/base/fixtures/default_oauth_apps_docker.json b/geonode/base/fixtures/default_oauth_apps_docker.json new file mode 100644 index 0000000..975680f --- /dev/null +++ b/geonode/base/fixtures/default_oauth_apps_docker.json @@ -0,0 +1,21 @@ +[ +{ + "model": "oauth2_provider.application", + "pk": 1001, + "fields": { + "skip_authorization": true, + "created": "2018-05-31T10:00:31.661Z", + "updated": "2018-05-31T11:30:31.245Z", + "algorithm": "RS256", + "redirect_uris": "http://geonode/geoserver", + "name": "GeoServer", + "authorization_grant_type": "authorization-code", + "client_type": "confidential", + "client_id": "Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV", + "client_secret": "rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3", + "user": [ + "admin" + ] + } +} +] diff --git a/geonode/base/fixtures/group_test_data.json b/geonode/base/fixtures/group_test_data.json new file mode 100644 index 0000000..90d4822 --- /dev/null +++ b/geonode/base/fixtures/group_test_data.json @@ -0,0 +1,78 @@ +[ + { + "pk": 1, + "model": "auth.group", + "fields": { + "name": "bar", + "permissions": [] + } + }, + { + "fields": { + "access": "public", + "description": "baz", + "logo": "people_peoplegroup/cactuar_1.jpg", + "slug": "bar", + "last_modified": "2011-06-09 15:45:34", + "created": "2011-06-09 15:45:34", + "group": 1, + "title": "bar" + }, + "model": "groups.groupprofile", + "pk": 2 + }, + { + "fields": { + "date_joined": "2011-06-09 15:15:27", + "email": "ad@m.in", + "first_name": "admin", + "groups": [], + "is_active": true, + "is_staff": true, + "is_superuser": true, + "last_login": "2011-06-09 15:45:34", + "last_name": "", + "password": "pbkdf2_sha256$30000$rjuGt0Obn8on$cxF75frIOSaitNklLZ0IJ/VonUW0fwEFVF96o0M+lGc=", + "user_permissions": [], + "username": "admin" + }, + "model": "people.profile", + "pk": 1 + }, + { + "fields": { + "date_joined": "2011-06-09 15:46:03", + "email": "", + "first_name": "Norman", + "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": "2011-06-09 15:46:03", + "last_name": "Normandy", + "password": "sha1$b0022$1592757da52adeed741051a203fe2d65b6798e83", + "user_permissions": [], + "username": "norman" + }, + "model": "people.profile", + "pk": 2 + }, + { + "fields": { + "date_joined": "2011-06-09 15:46:03", + "email": "", + "first_name": "Test", + "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": "2011-06-09 15:46:03", + "last_name": "User", + "password": "sha1$b0022$1592757da52adeed741051a203fe2d65b6798e83", + "user_permissions": [], + "username": "test_user" + }, + "model": "people.profile", + "pk": 3 + } +] diff --git a/geonode/base/fixtures/initial_data.json b/geonode/base/fixtures/initial_data.json new file mode 100644 index 0000000..d5333bc --- /dev/null +++ b/geonode/base/fixtures/initial_data.json @@ -0,0 +1,4877 @@ +[ + { + "pk": 1, + "model": "base.topiccategory", + "fields": { + "description": "Information pertaining to earth sciences. Examples: geophysical features and processes, geology, minerals, sciences dealing with the composition, structure and origin of the earth s rocks, risks of earthquakes, volcanic activity, landslides, gravity information, soils, permafrost, hydrogeology, erosion.", + "gn_description": "Geoscientific Information", + "is_choice": true, + "fa_class": "fa-bullseye", + "identifier": "geoscientificInformation" + } + }, + { + "pk": 2, + "model": "base.topiccategory", + "fields": { + "description": "Rearing of animals and/or cultivation of plants. Examples: agriculture, irrigation, aquaculture, plantations, herding, pests and diseases affecting crops and livestock.", + "gn_description": "Farming", + "is_choice": true, + "fa_class": "fa-lemon-o", + "identifier": "farming" + } + }, + { + "pk": 3, + "model": "base.topiccategory", + "fields": { + "description": "Height above or below sea level. Examples: altitude, bathymetry, digital elevation models, slope, derived products.", + "gn_description": "Elevation", + "is_choice": true, + "fa_class": "fa-flag", + "identifier": "elevation" + } + }, + { + "pk": 4, + "model": "base.topiccategory", + "fields": { + "description": "Energy, water and waste systems and communications infrastructure and services. Examples: hydroelectricity, geothermal, solar and nuclear sources of energy, water purification and distribution, sewage collection and disposal, electricity and gas distribution, data communication, telecommunication, radio, communication networks.", + "gn_description": "Utilities Communication", + "is_choice": true, + "fa_class": "fa-phone", + "identifier": "utilitiesCommunication" + } + }, + { + "pk": 5, + "model": "base.topiccategory", + "fields": { + "description": "Features and characteristics of salt water bodies (excluding inland waters). Examples: tides, tidal waves, coastal information, reefs.", + "gn_description": "Oceans", + "is_choice": true, + "fa_class": "fa-anchor", + "identifier": "oceans" + } + }, + { + "pk": 6, + "model": "base.topiccategory", + "fields": { + "description": "Legal land descriptions. Examples: political and administrative boundaries.", + "gn_description": "Boundaries", + "is_choice": true, + "fa_class": "fa-ellipsis-h", + "identifier": "boundaries" + } + }, + { + "pk": 7, + "model": "base.topiccategory", + "fields": { + "description": "Inland water features, drainage systems and their characteristics. Examples: rivers and glaciers, salt lakes, water utilization plans, dams, currents, floods, water quality, hydrographic charts.", + "gn_description": "Inland Waters", + "is_choice": true, + "fa_class": "fa-tint", + "identifier": "inlandWaters" + } + }, + { + "pk": 8, + "model": "base.topiccategory", + "fields": { + "description": "Military bases, structures, activities. Examples: barracks, training grounds, military transportation, information collection.", + "gn_description": "Intelligence Military", + "is_choice": true, + "fa_class": "fa-fighter-jet", + "identifier": "intelligenceMilitary" + } + }, + { + "pk": 9, + "model": "base.topiccategory", + "fields": { + "description": "Environmental resources, protection and conservation. Examples: environmental pollution, waste storage and treatment, environmental impact assessment, monitoring environmental risk, nature reserves, landscape.", + "gn_description": "Environment", + "is_choice": true, + "fa_class": "fa-tree", + "identifier": "environment" + } + }, + { + "pk": 10, + "model": "base.topiccategory", + "fields": { + "description": "Positional information and services. Examples: addresses, geodetic networks, control points, postal zones and services, place names.", + "gn_description": "Location", + "is_choice": true, + "fa_class": "fa-map-marker", + "identifier": "location" + } + }, + { + "pk": 11, + "model": "base.topiccategory", + "fields": { + "description": "Economic activities, conditions and employment. Examples: production, labour, revenue, commerce, industry, tourism and ecotourism, forestry, fisheries, commercial or subsistence hunting, exploration and exploitation of resources such as minerals, oil and gas.", + "gn_description": "Economy", + "is_choice": true, + "fa_class": "fa-shopping-cart", + "identifier": "economy" + } + }, + { + "pk": 12, + "model": "base.topiccategory", + "fields": { + "description": "Information used for appropriate actions for future use of the land. Examples: land use maps, zoning maps, cadastral surveys, land ownership.", + "gn_description": "Planning Cadastre", + "is_choice": true, + "fa_class": "fa-home", + "identifier": "planningCadastre" + } + }, + { + "pk": 13, + "model": "base.topiccategory", + "fields": { + "description": "Flora and/or fauna in natural environment. Examples: wildlife, vegetation, biological sciences, ecology, wilderness, sealife, wetlands, habitat.", + "gn_description": "Biota", + "is_choice": true, + "fa_class": "fa-leaf", + "identifier": "biota" + } + }, + { + "pk": 14, + "model": "base.topiccategory", + "fields": { + "description": "Health, health services, human ecology, and safety. Examples: disease and illness, factors affecting health, hygiene, substance abuse, mental and physical health, health services.", + "gn_description": "Health", + "is_choice": true, + "fa_class": "fa-stethoscope", + "identifier": "health" + } + }, + { + "pk": 15, + "model": "base.topiccategory", + "fields": { + "description": "Base maps. Examples: land cover, topographic maps, imagery, unclassified images, annotations.", + "gn_description": "Imagery Base Maps Earth Cover", + "is_choice": true, + "fa_class": "fa-globe", + "identifier": "imageryBaseMapsEarthCover" + } + }, + { + "pk": 16, + "model": "base.topiccategory", + "fields": { + "description": "Means and aids for conveying persons and/or goods. Examples: roads, airports/airstrips, shipping routes, tunnels, nautical charts, vehicle or vessel location, aeronautical charts, railways.", + "gn_description": "Transportation", + "is_choice": true, + "fa_class": "fa-truck", + "identifier": "transportation" + } + }, + { + "pk": 17, + "model": "base.topiccategory", + "fields": { + "description": "Characteristics of society and cultures. Examples: settlements, anthropology, archaeology, education, traditional beliefs, manners and customs, demographic data, recreational areas and activities, social impact assessments, crime and justice, census information.", + "gn_description": "Society", + "is_choice": true, + "fa_class": "fa-comments", + "identifier": "society" + } + }, + { + "pk": 18, + "model": "base.topiccategory", + "fields": { + "description": "Man-made construction. Examples: buildings, museums, churches, factories, housing, monuments, shops, towers.", + "gn_description": "Structure", + "is_choice": true, + "fa_class": "fa-building", + "identifier": "structure" + } + }, + { + "pk": 19, + "model": "base.topiccategory", + "fields": { + "description": "Processes and phenomena of the atmosphere. Examples: cloud cover, weather, climate, atmospheric conditions, climate change, precipitation.", + "gn_description": "Climatology Meteorology Atmosphere", + "is_choice": true, + "fa_class": "fa-cloud", + "identifier": "climatologyMeteorologyAtmosphere" + } + }, + { + "pk": 20, + "model": "base.topiccategory", + "fields": { + "description": "Settlements, anthropology, archaeology, education, traditional beliefs, manners and customs, demographic data, recreational areas and activities, social impact assessments, crime and justice, census information. Economic activities, conditions and employment.", + "gn_description": "Population", + "is_choice": true, + "fa_class": "fa-male", + "identifier": "population" + } + }, + { + "pk": 1, + "model": "base.spatialrepresentationtype", + "fields": { + "is_choice": true, + "gn_description": "vector data is used to represent geographic data", + "identifier": "vector", + "description": "vector data is used to represent geographic data" + } + }, + { + "pk": 2, + "model": "base.spatialrepresentationtype", + "fields": { + "is_choice": true, + "gn_description": "grid data is used to represent geographic data", + "identifier": "grid", + "description": "grid data is used to represent geographic data" + } + }, + { + "pk": 3, + "model": "base.spatialrepresentationtype", + "fields": { + "is_choice": true, + "gn_description": "textual or tabular data is used to represent geographic data", + "identifier": "textTable", + "description": "textual or tabular data is used to represent geographic data" + } + }, + { + "pk": 4, + "model": "base.spatialrepresentationtype", + "fields": { + "is_choice": true, + "gn_description": "triangulated irregular network", + "identifier": "tin", + "description": "triangulated irregular network" + } + }, + { + "pk": 5, + "model": "base.spatialrepresentationtype", + "fields": { + "is_choice": true, + "gn_description": "three-dimensional view formed by the intersecting homologous rays of an overlapping pair of images", + "identifier": "stereoModel", + "description": "three-dimensional view formed by the intersecting homologous rays of an overlapping pair of images" + } + }, + { + "pk": 6, + "model": "base.spatialrepresentationtype", + "fields": { + "is_choice": true, + "gn_description": "scene from a video recording", + "identifier": "video", + "description": "scene from a video recording" + } + }, + { + "pk": 1, + "model": "base.license", + "fields": { + "identifier": "not_specified", + "name":"Not Specified", + "abbreviation":"", + "description":"The original author did not specify a license.", + "url":"", + "license_text":"Not applicable" + } + }, +{ + "pk": 2, + "model": "base.license", + "fields": { + "identifier":"varied_original", + "name":"Varied / Original", + "abbreviation":"", + "description":"This item is either licensed under multiple licenses. See the item's abstract for more information or contact the distributor.", + "url":"", + "license_text":"Not applicable" + } + }, + { + "pk": 3, + "model": "base.license", + "fields": { + "identifier":"varied_derived", + "name":"Varied / Derived", + "abbreviation":"", + "description":"The constituent parts of this item have different licenses. Go to each part to see license information.", + "url":"", + "license_text":"Not applicable" + } + }, + { + "pk": 4, + "model": "base.license", + "fields": { + "identifier":"public_domain", + "name":"Public Domain", + "abbreviation":"PD", + "description":"Works in the public domain may be used freely without the permission of the former copyright owner.", + "url":"http://www.copyright.gov/help/faq/faq-definitions.html", + "license_text":"The public domain is not a place. A work of authorship is in the “public domain” if it is no longer under copyright protection or if it failed to meet the requirements for copyright protection. Works in the public domain may be used freely without the permission of the former copyright owner." + } + }, + { + "pk": 5, + "model": "base.license", + "fields": { + "identifier":"public_domain_usg", + "name":"Public Domain / USG", + "abbreviation":"PD/USG", + "description":"This project constitutes a work of the United States Government and is not subject to domestic copyright protection under 17 USC § 105.", + "url":"https://raw.githubusercontent.com/state-hiu/cybergis-licenses/master/licenses/PUBLICDOMAIN-LICENSE-RAW.txt", + "license_text":"This project constitutes a work of the United States Government and is not subject to domestic copyright protection under 17 USC § 105." + } + }, + { + "pk": 6, + "model": "base.license", + "fields": { + "identifier":"odbl", + "name":"Open Data Commons Open Database License / OSM", + "abbreviation":"ODbL/OSM", + "description":"You are free to copy, distribute, transmit and adapt our data, as long as you credit OpenStreetMap and its contributors\nIf you alter or build upon our data, you may distribute the result only under the same licence.", + "url":"http://www.openstreetmap.org/copyright", + "license_text":"" + } + }, + { + "pk": 7, + "model": "base.license", + "fields": { + "identifier":"nextview", + "name":"NextView", + "abbreviation":"NV", + "description":"This data is licensed for use by the US Government (USG) under the NextView (NV) license and copyrighted by Digital Globe or GeoEye. The NV license allows the USG to share the imagery and Literal Imagery Derived Products (LIDP) with entities outside the USG when that entity is working directly with the USG, for the USG, or in a manner that is directly beneficial to the USG. The party receiving the data can only use the imagery or LIDP for the original purpose or only as otherwise agreed to by the USG. The party receiving the data cannot share the imagery or LIDP with a third party without express permission from the USG. At no time should this imagery or LIDP be used for other than USG-related purposes and must not be used for commercial gain. The copyright information should be maintained at all times. Your acceptance of these license terms is implied by your use.\nIn other words, you may only use NextView imagery linked from this site for digitizing OpenStreetMap data for humanitarian purposes.", + "url":"https://raw.githubusercontent.com/state-hiu/cybergis-licenses/master/licenses/NEXTVIEW-LICENSE-RAW.txt", + "license_text":"" + } + }, + { + "pk": 1, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "exclusive right to the publication, production, or sale of the rights to a literary, dramatic, musical, or artistic work, or to the use of a commercial print or label, granted by law for a specified period of time to an author, composer, artist, distributor", + "identifier": "copyright", + "description": "exclusive right to the publication, production, or sale of the rights to a literary, dramatic, musical, or artistic work, or to the use of a commercial print or label, granted by law for a specified period of time to an author, composer, artist, distributor" + } + }, + { + "pk": 6, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "rights to financial benefit from and control of distribution of non-tangible property that is a result of creativity", + "identifier": "intellectualPropertyRights", + "description": "rights to financial benefit from and control of distribution of non-tangible property that is a result of creativity" + } + }, + { + "pk": 5, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "formal permission to do something", + "identifier": "license", + "description": "formal permission to do something" + } + }, + { + "pk": 8, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "limitation not listed", + "identifier": "otherRestrictions", + "description": "limitation not listed" + } + }, + { + "pk": 2, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "government has granted exclusive right to make, sell, use or license an invention or discovery", + "identifier": "patent", + "description": "government has granted exclusive right to make, sell, use or license an invention or discovery" + } + }, + { + "pk": 3, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "produced or sold information awaiting a patent", + "identifier": "patentPending", + "description": "produced or sold information awaiting a patent" + } + }, + { + "pk": 7, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "withheld from general circulation or disclosure", + "identifier": "restricted", + "description": "withheld from general circulation or disclosure" + } + }, + { + "pk": 4, + "model": "base.restrictioncodetype", + "fields": { + "is_choice": true, + "gn_description": "a name, symbol, or other device identifying a product, officially registered and legally restricted to the use of the owner or manufacturer", + "identifier": "trademark", + "description": "a name, symbol, or other device identifying a product, officially registered and legally restricted to the use of the owner or manufacturer" + } + }, + { + "pk": 1, + "model": "base.region", + "fields": { + "rght": 516, + "code": "GLO", + "name": "Global", + "parent": null, + "level": 0, + "lft": 1, + "tree_id": 90, + "bbox_x0": -180, + "bbox_x1": 180, + "bbox_y0": -90, + "bbox_y1": 90 + } + }, + { + "pk": 2, + "model": "base.region", + "fields": { + "rght": 212, + "code": "NAM", + "name": "North America", + "parent": 254, + "level": 2, + "lft": 203, + "tree_id": 90, + "bbox_x0": -167.276413, + "bbox_x1": -52.23304, + "bbox_y0": 5.49955, + "bbox_y1": 83.162102 + } + }, + { + "pk": 3, + "model": "base.region", + "fields": { + "rght": 202, + "code": "CAM", + "name": "Central America", + "parent": 254, + "level": 2, + "lft": 187, + "tree_id": 90, + "bbox_x0": -118.867172, + "bbox_x1": -66.869827, + "bbox_y0": -4.23048, + "bbox_y1": 32.71862 + } + }, + { + "pk": 4, + "model": "base.region", + "fields": { + "rght": 242, + "code": "SAM", + "name": "South America", + "parent": 254, + "level": 2, + "lft": 213, + "tree_id": 90, + "bbox_x0": -109.47493, + "bbox_x1": -26.33247, + "bbox_y0": -59.450451, + "bbox_y1": 13.39029 + } + }, + { + "pk": 5, + "model": "base.region", + "fields": { + "rght": 433, + "code": "EUR", + "name": "Europe", + "parent": null, + "level": 1, + "lft": 318, + "tree_id": 90, + "bbox_x0": -31.266001, + "bbox_x1": 39.869301, + "bbox_y0": 27.636311, + "bbox_y1": 81.008797 + } + }, + { + "pk": 6, + "model": "base.region", + "fields": { + "rght": 317, + "code": "ASI", + "name": "Asia", + "parent": null, + "level": 1, + "lft": 246, + "tree_id": 90, + "bbox_x0": 19.6381, + "bbox_x1": 180, + "bbox_y0": -12.56111, + "bbox_y1": 82.50045 + } + }, + { + "pk": 7, + "model": "base.region", + "fields": { + "rght": 316, + "code": "SEA", + "name": "Southeast Asia", + "parent": 6, + "level": 2, + "lft": 293, + "tree_id": 90, + "bbox_x0": 68.0327, + "bbox_x1": 141.021805, + "bbox_y0": -12.56111, + "bbox_y1": 35.504211 + } + }, + { + "pk": 8, + "model": "base.region", + "fields": { + "rght": 260, + "code": "CTA", + "name": "Central Asia", + "parent": 6, + "level": 2, + "lft": 247, + "tree_id": 90, + "bbox_x0": 44.236641, + "bbox_x1": 90.076767, + "bbox_y0": 33.890511, + "bbox_y1": 54.845139 + } + }, + { + "pk": 9, + "model": "base.region", + "fields": { + "rght": 292, + "code": "SAS", + "name": "South Asia", + "parent": 6, + "level": 2, + "lft": 277, + "tree_id": 90, + "bbox_x0": 19.6381, + "bbox_x1": 180, + "bbox_y0": -12.56111, + "bbox_y1": 82.50045 + } + }, + { + "pk": 10, + "model": "base.region", + "fields": { + "rght": 127, + "code": "AFR", + "name": "Africa", + "parent": null, + "level": 1, + "lft": 2, + "tree_id": 90, + "bbox_x0": -25.35874, + "bbox_x1": 63.525379, + "bbox_y0": -46.900452, + "bbox_y1": 37.56712 + } + }, + { + "pk": 11, + "model": "base.region", + "fields": { + "rght": 64, + "code": "NAF", + "name": "North Africa", + "parent": 10, + "level": 2, + "lft": 49, + "tree_id": 90, + "bbox_x0": -17.10317, + "bbox_x1": 38.833801, + "bbox_y0": 3.48639, + "bbox_y1": 37.56712 + } + }, + { + "pk": 12, + "model": "base.region", + "fields": { + "rght": 48, + "code": "EAF", + "name": "East Africa", + "parent": 10, + "level": 2, + "lft": 13, + "tree_id": 90, + "bbox_x0": 22.855089, + "bbox_x1": 63.94656, + "bbox_y0": -25.84763, + "bbox_y1": 17.467039 + } + }, + { + "pk": 13, + "model": "base.region", + "fields": { + "rght": 126, + "code": "WAF", + "name": "West Africa", + "parent": 10, + "level": 2, + "lft": 83, + "tree_id": 90, + "bbox_x0": -26.758421, + "bbox_x1": 24.002661, + "bbox_y0": -9.29925, + "bbox_y1": 27.702801 + } + }, + { + "pk": 14, + "model": "base.region", + "fields": { + "rght": 82, + "code": "SAF", + "name": "Southern Africa", + "parent": 10, + "level": 2, + "lft": 65, + "tree_id": 90, + "bbox_x0": 8.93107, + "bbox_x1": 42.74847, + "bbox_y0": -35.507481, + "bbox_y1": -13.27553 + } + }, + { + "pk": 15, + "model": "base.region", + "fields": { + "rght": 463, + "code": "MES", + "name": "Middle East", + "parent": null, + "level": 1, + "lft": 434, + "tree_id": 90, + "bbox_x0": 24.698099, + "bbox_x1": 63.317459, + "bbox_y0": 12.111, + "bbox_y1": 42.10751 + } + }, + { + "pk": 16, + "model": "base.region", + "fields": { + "rght": 245, + "code": "ANT", + "name": "Antarctica", + "parent": null, + "level": 1, + "lft": 244, + "tree_id": 90, + "bbox_x0": -180, + "bbox_x1": 180, + "bbox_y0": -90, + "bbox_y1": -73 + } + }, + { + "pk": 17, + "model": "base.region", + "fields": { + "rght": 249, + "code": "AFG", + "name": "Afghanistan", + "parent": 8, + "level": 3, + "lft": 248, + "tree_id": 90, + "bbox_x0": 60.478439, + "bbox_x1": 74.879433, + "bbox_y0": 29.37747, + "bbox_y1": 38.483421 + } + }, + { + "pk": 18, + "model": "base.region", + "fields": { + "rght": 320, + "code": "ALA", + "name": "Aland Islands", + "parent": 5, + "level": 2, + "lft": 319, + "tree_id": 90, + "bbox_x0": 19.262711, + "bbox_x1": 21.324409, + "bbox_y0": 59.736301, + "bbox_y1": 60.665581 + } + }, + { + "pk": 19, + "model": "base.region", + "fields": { + "rght": 322, + "code": "ALB", + "name": "Albania", + "parent": 5, + "level": 2, + "lft": 321, + "tree_id": 90, + "bbox_x0": 19.28219, + "bbox_x1": 21.057819, + "bbox_y0": 39.644489, + "bbox_y1": 42.660801 + } + }, + { + "pk": 20, + "model": "base.region", + "fields": { + "rght": 51, + "code": "DZA", + "name": "Algeria", + "parent": 11, + "level": 3, + "lft": 50, + "tree_id": 90, + "bbox_x0": -8.67386, + "bbox_x1": 11.97955, + "bbox_y0": 18.96002, + "bbox_y1": 37.093731 + } + }, + { + "pk": 21, + "model": "base.region", + "fields": { + "rght": 466, + "code": "ASM", + "name": "American Samoa", + "parent": 256, + "level": 2, + "lft": 465, + "tree_id": 90, + "bbox_x0": -171.091873, + "bbox_x1": -169.416077, + "bbox_y0": -14.38247, + "bbox_y1": -11.04969 + } + }, + { + "pk": 22, + "model": "base.region", + "fields": { + "rght": 324, + "code": "AND", + "name": "Andorra", + "parent": 5, + "level": 2, + "lft": 323, + "tree_id": 90, + "bbox_x0": 1.41382, + "bbox_x1": 1.78659, + "bbox_y0": 42.42873, + "bbox_y1": 42.65601 + } + }, + { + "pk": 23, + "model": "base.region", + "fields": { + "rght": 85, + "code": "AGO", + "name": "Angola", + "parent": 13, + "level": 3, + "lft": 84, + "tree_id": 90, + "bbox_x0": 11.6792, + "bbox_x1": 24.082109, + "bbox_y0": -18.04207, + "bbox_y1": -4.37259 + } + }, + { + "pk": 24, + "model": "base.region", + "fields": { + "rght": 131, + "code": "AIA", + "name": "Anguilla", + "parent": 255, + "level": 3, + "lft": 130, + "tree_id": 90, + "bbox_x0": -63.434872, + "bbox_x1": -62.916199, + "bbox_y0": 18.149549, + "bbox_y1": 18.61278 + } + }, + { + "pk": 25, + "model": "base.region", + "fields": { + "rght": 133, + "code": "ATG", + "name": "Antigua and Barbuda", + "parent": 255, + "level": 3, + "lft": 132, + "tree_id": 90, + "bbox_x0": -62.352402, + "bbox_x1": -61.659081, + "bbox_y0": 16.927219, + "bbox_y1": 17.72938 + } + }, + { + "pk": 26, + "model": "base.region", + "fields": { + "rght": 215, + "code": "ARG", + "name": "Argentina", + "parent": 4, + "level": 3, + "lft": 214, + "tree_id": 90, + "bbox_x0": -73.577782, + "bbox_x1": -53.637539, + "bbox_y0": -55.057362, + "bbox_y1": -21.78126 + } + }, + { + "pk": 27, + "model": "base.region", + "fields": { + "rght": 326, + "code": "ARM", + "name": "Armenia", + "parent": 5, + "level": 2, + "lft": 325, + "tree_id": 90, + "bbox_x0": 43.449749, + "bbox_x1": 46.630039, + "bbox_y0": 38.830521, + "bbox_y1": 41.30183 + } + }, + { + "pk": 28, + "model": "base.region", + "fields": { + "rght": 135, + "code": "ABW", + "name": "Aruba", + "parent": 255, + "level": 3, + "lft": 134, + "tree_id": 90, + "bbox_x0": -70.0611, + "bbox_x1": -69.8669, + "bbox_y0": 12.4061, + "bbox_y1": 12.6306 + } + }, + { + "pk": 29, + "model": "base.region", + "fields": { + "rght": 468, + "code": "AUS", + "name": "Australia", + "parent": 256, + "level": 2, + "lft": 467, + "tree_id": 90, + "bbox_x0": 112.921112, + "bbox_x1": 159.278717, + "bbox_y0": -54.640301, + "bbox_y1": -9.22882 + } + }, + { + "pk": 30, + "model": "base.region", + "fields": { + "rght": 328, + "code": "AUT", + "name": "Austria", + "parent": 5, + "level": 2, + "lft": 327, + "tree_id": 90, + "bbox_x0": 9.53079, + "bbox_x1": 17.160749, + "bbox_y0": 46.372299, + "bbox_y1": 49.02071 + } + }, + { + "pk": 31, + "model": "base.region", + "fields": { + "rght": 330, + "code": "AZE", + "name": "Azerbaijan", + "parent": 5, + "level": 2, + "lft": 329, + "tree_id": 90, + "bbox_x0": 44.7719, + "bbox_x1": 50.6078, + "bbox_y0": 38.3970, + "bbox_y1": 41.9056 + } + }, + { + "pk": 32, + "model": "base.region", + "fields": { + "rght": 137, + "code": "BHS", + "name": "Bahamas", + "parent": 255, + "level": 3, + "lft": 136, + "tree_id": 90, + "bbox_x0": -80.499229, + "bbox_x1": -72.649513, + "bbox_y0": 20.916059, + "bbox_y1": 27.933781 + } + }, + { + "pk": 33, + "model": "base.region", + "fields": { + "rght": 436, + "code": "BHR", + "name": "Bahrain", + "parent": 15, + "level": 2, + "lft": 435, + "tree_id": 90, + "bbox_x0": 50.385799, + "bbox_x1": 50.828499, + "bbox_y0": 25.5422, + "bbox_y1": 26.292391 + } + }, + { + "pk": 34, + "model": "base.region", + "fields": { + "rght": 279, + "code": "BGD", + "name": "Bangladesh", + "parent": 9, + "level": 3, + "lft": 278, + "tree_id": 90, + "bbox_x0": 88.028198, + "bbox_x1": 92.673599, + "bbox_y0": 20.585199, + "bbox_y1": 26.631701 + } + }, + { + "pk": 35, + "model": "base.region", + "fields": { + "rght": 139, + "code": "BRB", + "name": "Barbados", + "parent": 255, + "level": 3, + "lft": 138, + "tree_id": 90, + "bbox_x0": -59.648918, + "bbox_x1": -59.420368, + "bbox_y0": 13.03984, + "bbox_y1": 13.32725 + } + }, + { + "pk": 36, + "model": "base.region", + "fields": { + "rght": 332, + "code": "BLR", + "name": "Belarus", + "parent": 5, + "level": 2, + "lft": 331, + "tree_id": 90, + "bbox_x0": 23.17679, + "bbox_x1": 32.77071, + "bbox_y0": 51.256401, + "bbox_y1": 56.16571 + } + }, + { + "pk": 37, + "model": "base.region", + "fields": { + "rght": 334, + "code": "BEL", + "name": "Belgium", + "parent": 5, + "level": 2, + "lft": 333, + "tree_id": 90, + "bbox_x0": 2.54563, + "bbox_x1": 6.40791, + "bbox_y0": 49.496899, + "bbox_y1": 51.505081 + } + }, + { + "pk": 38, + "model": "base.region", + "fields": { + "rght": 189, + "code": "BLZ", + "name": "Belize", + "parent": 3, + "level": 3, + "lft": 188, + "tree_id": 90, + "bbox_x0": -89.224823, + "bbox_x1": -87.468132, + "bbox_y0": 15.8893, + "bbox_y1": 18.49655 + } + }, + { + "pk": 39, + "model": "base.region", + "fields": { + "rght": 87, + "code": "BEN", + "name": "Benin", + "parent": 13, + "level": 3, + "lft": 86, + "tree_id": 90, + "bbox_x0": 0.77456, + "bbox_x1": 3.8517, + "bbox_y0": 6.22574, + "bbox_y1": 12.41834 + } + }, + { + "pk": 40, + "model": "base.region", + "fields": { + "rght": 141, + "code": "BMU", + "name": "Bermuda", + "parent": 255, + "level": 3, + "lft": 140, + "tree_id": 90, + "bbox_x0": -64.896042, + "bbox_x1": -64.642952, + "bbox_y0": 32.230709, + "bbox_y1": 32.393829 + } + }, + { + "pk": 41, + "model": "base.region", + "fields": { + "rght": 281, + "code": "BTN", + "name": "Bhutan", + "parent": 9, + "level": 3, + "lft": 280, + "tree_id": 90, + "bbox_x0": 88.759521, + "bbox_x1": 92.125023, + "bbox_y0": 26.7075, + "bbox_y1": 28.3235 + } + }, + { + "pk": 42, + "model": "base.region", + "fields": { + "rght": 217, + "code": "BOL", + "name": "Bolivia", + "parent": 4, + "level": 3, + "lft": 216, + "tree_id": 90, + "bbox_x0": -69.640762, + "bbox_x1": -57.458092, + "bbox_y0": -22.89613, + "bbox_y1": -9.68056 + } + }, + { + "pk": 43, + "model": "base.region", + "fields": { + "rght": 336, + "code": "BIH", + "name": "Bosnia and Herzegovina", + "parent": 5, + "level": 2, + "lft": 335, + "tree_id": 90, + "bbox_x0": 15.74909, + "bbox_x1": 19.62907, + "bbox_y0": 42.56451, + "bbox_y1": 45.276001 + } + }, + { + "pk": 44, + "model": "base.region", + "fields": { + "rght": 67, + "code": "BWA", + "name": "Botswana", + "parent": 14, + "level": 3, + "lft": 66, + "tree_id": 90, + "bbox_x0": 19.999531, + "bbox_x1": 29.360781, + "bbox_y0": -26.90724, + "bbox_y1": -17.780809 + } + }, + { + "pk": 45, + "model": "base.region", + "fields": { + "rght": 219, + "code": "BRA", + "name": "Brazil", + "parent": 4, + "level": 3, + "lft": 218, + "tree_id": 90, + "bbox_x0": -73.985527, + "bbox_x1": -28.839041, + "bbox_y0": -33.750702, + "bbox_y1": 5.26486 + } + }, + { + "pk": 46, + "model": "base.region", + "fields": { + "rght": 143, + "code": "VGB", + "name": "British Virgin Islands", + "parent": 255, + "level": 3, + "lft": 142, + "tree_id": 90, + "bbox_x0": -64.783012, + "bbox_x1": -64.268761, + "bbox_y0": 18.312731, + "bbox_y1": 18.757219 + } + }, + { + "pk": 47, + "model": "base.region", + "fields": { + "rght": 295, + "code": "BRN", + "name": "Brunei Darussalam", + "parent": 7, + "level": 3, + "lft": 294, + "tree_id": 90, + "bbox_x0": 114.071457, + "bbox_x1": 115.359451, + "bbox_y0": 4.00309, + "bbox_y1": 5.04717 + } + }, + { + "pk": 48, + "model": "base.region", + "fields": { + "rght": 338, + "code": "BGR", + "name": "Bulgaria", + "parent": 5, + "level": 2, + "lft": 337, + "tree_id": 90, + "bbox_x0": 22.35741, + "bbox_x1": 28.60882, + "bbox_y0": 41.235931, + "bbox_y1": 44.227261 + } + }, + { + "pk": 49, + "model": "base.region", + "fields": { + "rght": 89, + "code": "BFA", + "name": "Burkina Faso", + "parent": 13, + "level": 3, + "lft": 88, + "tree_id": 90, + "bbox_x0": -5.51891, + "bbox_x1": 2.40539, + "bbox_y0": 9.4011, + "bbox_y1": 15.08259 + } + }, + { + "pk": 50, + "model": "base.region", + "fields": { + "rght": 15, + "code": "BDI", + "name": "Burundi", + "parent": 12, + "level": 3, + "lft": 14, + "tree_id": 90, + "bbox_x0": 28.993071, + "bbox_x1": 30.847719, + "bbox_y0": -4.46571, + "bbox_y1": -2.31012 + } + }, + { + "pk": 51, + "model": "base.region", + "fields": { + "rght": 297, + "code": "KHM", + "name": "Cambodia", + "parent": 7, + "level": 3, + "lft": 296, + "tree_id": 90, + "bbox_x0": 102.340012, + "bbox_x1": 107.627724, + "bbox_y0": 9.28325, + "bbox_y1": 14.6864 + } + }, + { + "pk": 52, + "model": "base.region", + "fields": { + "rght": 91, + "code": "CMR", + "name": "Cameroon", + "parent": 13, + "level": 3, + "lft": 90, + "tree_id": 90, + "bbox_x0": 8.49477, + "bbox_x1": 16.19211, + "bbox_y0": 1.65254, + "bbox_y1": 13.07805 + } + }, + { + "pk": 53, + "model": "base.region", + "fields": { + "rght": 205, + "code": "CAN", + "name": "Canada", + "parent": 2, + "level": 3, + "lft": 204, + "tree_id": 90, + "bbox_x0": -141.002701, + "bbox_x1": -52.620201, + "bbox_y0": 41.681019, + "bbox_y1": 83.110619 + } + }, + { + "pk": 54, + "model": "base.region", + "fields": { + "rght": 93, + "code": "CPV", + "name": "Cape Verde", + "parent": 13, + "level": 3, + "lft": 92, + "tree_id": 90, + "bbox_x0": -25.35874, + "bbox_x1": -22.666201, + "bbox_y0": 14.80221, + "bbox_y1": 17.19717 + } + }, + { + "pk": 55, + "model": "base.region", + "fields": { + "rght": 145, + "code": "CYM", + "name": "Cayman Islands", + "parent": 255, + "level": 3, + "lft": 144, + "tree_id": 90, + "bbox_x0": -81.420593, + "bbox_x1": -79.722321, + "bbox_y0": 19.262659, + "bbox_y1": 19.75738 + } + }, + { + "pk": 56, + "model": "base.region", + "fields": { + "rght": 5, + "code": "CAF", + "name": "Central African Republic", + "parent": 257, + "level": 3, + "lft": 4, + "tree_id": 90, + "bbox_x0": 14.42009, + "bbox_x1": 27.463421, + "bbox_y0": 2.22051, + "bbox_y1": 11.00756 + } + }, + { + "pk": 57, + "model": "base.region", + "fields": { + "rght": 7, + "code": "TCD", + "name": "Chad", + "parent": 257, + "level": 3, + "lft": 6, + "tree_id": 90, + "bbox_x0": 13.47592, + "bbox_x1": 24.00161, + "bbox_y0": 7.44237, + "bbox_y1": 23.478239 + } + }, + { + "pk": 58, + "model": "base.region", + "fields": { + "rght": 340, + "code": "CIL", + "name": "Channel Islands", + "parent": 5, + "level": 2, + "lft": 339, + "tree_id": 90, + "bbox_x0": -2.67545, + "bbox_x1": -2.01129, + "bbox_y0": 49.16209, + "bbox_y1": 49.7393 + } + }, + { + "pk": 59, + "model": "base.region", + "fields": { + "rght": 221, + "code": "CHL", + "name": "Chile", + "parent": 4, + "level": 3, + "lft": 220, + "tree_id": 90, + "bbox_x0": -109.47493, + "bbox_x1": -66.417549, + "bbox_y0": -56.533779, + "bbox_y1": -17.507549 + } + }, + { + "pk": 60, + "model": "base.region", + "fields": { + "rght": 263, + "code": "CHN", + "name": "China", + "parent": 258, + "level": 3, + "lft": 262, + "tree_id": 90, + "bbox_x0": 73.557701, + "bbox_x1": 134.773605, + "bbox_y0": 15.77539, + "bbox_y1": 53.5606 + } + }, + { + "pk": 61, + "model": "base.region", + "fields": { + "rght": 265, + "code": "HKG", + "name": "China - Hong Kong", + "parent": 258, + "level": 3, + "lft": 264, + "tree_id": 90, + "bbox_x0": 113.835083, + "bbox_x1": 114.441788, + "bbox_y0": 22.153549, + "bbox_y1": 22.56204 + } + }, + { + "pk": 62, + "model": "base.region", + "fields": { + "rght": 267, + "code": "MAC", + "name": "China - Macao", + "parent": 258, + "level": 3, + "lft": 266, + "tree_id": 90, + "bbox_x0": 113.528351, + "bbox_x1": 113.598297, + "bbox_y0": 22.10977, + "bbox_y1": 22.21697 + } + }, + { + "pk": 63, + "model": "base.region", + "fields": { + "rght": 223, + "code": "COL", + "name": "Colombia", + "parent": 4, + "level": 3, + "lft": 222, + "tree_id": 90, + "bbox_x0": -81.728111, + "bbox_x1": -66.869827, + "bbox_y0": -4.23048, + "bbox_y1": 13.39029 + } + }, + { + "pk": 64, + "model": "base.region", + "fields": { + "rght": 17, + "code": "COM", + "name": "Comoros", + "parent": 12, + "level": 3, + "lft": 16, + "tree_id": 90, + "bbox_x0": 43.215778, + "bbox_x1": 44.538219, + "bbox_y0": -12.41382, + "bbox_y1": -11.36238 + } + }, + { + "pk": 65, + "model": "base.region", + "fields": { + "rght": 9, + "code": "COG", + "name": "Congo", + "parent": 257, + "level": 3, + "lft": 8, + "tree_id": 90, + "bbox_x0": 11.205, + "bbox_x1": 18.64983, + "bbox_y0": -5.02831, + "bbox_y1": 3.70308 + } + }, + { + "pk": 66, + "model": "base.region", + "fields": { + "rght": 470, + "code": "COK", + "name": "Cook Islands", + "parent": 256, + "level": 2, + "lft": 469, + "tree_id": 90, + "bbox_x0": -165.858093, + "bbox_x1": -157.312119, + "bbox_y0": -21.94416, + "bbox_y1": -8.94402 + } + }, + { + "pk": 67, + "model": "base.region", + "fields": { + "rght": 191, + "code": "CRI", + "name": "Costa Rica", + "parent": 3, + "level": 3, + "lft": 190, + "tree_id": 90, + "bbox_x0": -87.083778, + "bbox_x1": -82.556, + "bbox_y0": 5.49955, + "bbox_y1": 11.21681 + } + }, + { + "pk": 68, + "model": "base.region", + "fields": { + "rght": 95, + "code": "CIV", + "name": "Cote d'Ivoire", + "parent": 13, + "level": 3, + "lft": 94, + "tree_id": 90, + "bbox_x0": -8.6017249, + "bbox_x1": -2.4930309, + "bbox_y0": 4.1642077, + "bbox_y1": 10.740015 + } + }, + { + "pk": 69, + "model": "base.region", + "fields": { + "rght": 342, + "code": "HRV", + "name": "Croatia", + "parent": 5, + "level": 2, + "lft": 341, + "tree_id": 90, + "bbox_x0": 13.48972, + "bbox_x1": 19.44722, + "bbox_y0": 42.392208, + "bbox_y1": 46.554981 + } + }, + { + "pk": 70, + "model": "base.region", + "fields": { + "rght": 147, + "code": "CUB", + "name": "Cuba", + "parent": 255, + "level": 3, + "lft": 146, + "tree_id": 90, + "bbox_x0": -84.957428, + "bbox_x1": -74.131783, + "bbox_y0": 19.828079, + "bbox_y1": 23.283779 + } + }, + { + "pk": 71, + "model": "base.region", + "fields": { + "rght": 344, + "code": "CYP", + "name": "Cyprus", + "parent": 5, + "level": 2, + "lft": 343, + "tree_id": 90, + "bbox_x0": 32.27309, + "bbox_x1": 34.597919, + "bbox_y0": 34.563511, + "bbox_y1": 35.701542 + } + }, + { + "pk": 72, + "model": "base.region", + "fields": { + "rght": 346, + "code": "CZE", + "name": "Czech Republic", + "parent": 5, + "level": 2, + "lft": 345, + "tree_id": 90, + "bbox_x0": 12.0905901, + "bbox_x1": 18.859216, + "bbox_y0": 48.5518144, + "bbox_y1": 51.0557036 + } + }, + { + "pk": 73, + "model": "base.region", + "fields": { + "rght": 269, + "code": "PRK", + "name": "Democratic People's Republic of Korea", + "parent": 258, + "level": 3, + "lft": 268, + "tree_id": 90, + "bbox_x0": 124.182739, + "bbox_x1": 130.674713, + "bbox_y0": 37.632881, + "bbox_y1": 43.006001 + } + }, + { + "pk": 74, + "model": "base.region", + "fields": { + "rght": 11, + "code": "COD", + "name": "Democratic Republic of the Congo", + "parent": 257, + "level": 3, + "lft": 10, + "tree_id": 90, + "bbox_x0": 12.20663, + "bbox_x1": 31.30591, + "bbox_y0": -13.45567, + "bbox_y1": 5.38609 + } + }, + { + "pk": 75, + "model": "base.region", + "fields": { + "rght": 348, + "code": "DNK", + "name": "Denmark", + "parent": 5, + "level": 2, + "lft": 347, + "tree_id": 90, + "bbox_x0": 8.07472, + "bbox_x1": 15.19324, + "bbox_y0": 54.559132, + "bbox_y1": 57.751949 + } + }, + { + "pk": 76, + "model": "base.region", + "fields": { + "rght": 19, + "code": "DJI", + "name": "Djibouti", + "parent": 12, + "level": 3, + "lft": 18, + "tree_id": 90, + "bbox_x0": 41.773441, + "bbox_x1": 43.450459, + "bbox_y0": 10.90991, + "bbox_y1": 12.70683 + } + }, + { + "pk": 77, + "model": "base.region", + "fields": { + "rght": 149, + "code": "DMA", + "name": "Dominica", + "parent": 255, + "level": 3, + "lft": 148, + "tree_id": 90, + "bbox_x0": -61.4841, + "bbox_x1": -61.244148, + "bbox_y0": 15.20168, + "bbox_y1": 15.6318 + } + }, + { + "pk": 78, + "model": "base.region", + "fields": { + "rght": 151, + "code": "DOM", + "name": "Dominican Republic", + "parent": 255, + "level": 3, + "lft": 150, + "tree_id": 90, + "bbox_x0": -72.003479, + "bbox_x1": -68.319992, + "bbox_y0": 17.469299, + "bbox_y1": 19.92985 + } + }, + { + "pk": 79, + "model": "base.region", + "fields": { + "rght": 225, + "code": "ECU", + "name": "Ecuador", + "parent": 4, + "level": 3, + "lft": 224, + "tree_id": 90, + "bbox_x0": -91.66124, + "bbox_x1": -75.200073, + "bbox_y0": -5.01734, + "bbox_y1": 1.45421 + } + }, + { + "pk": 80, + "model": "base.region", + "fields": { + "rght": 53, + "code": "EGY", + "name": "Egypt", + "parent": 11, + "level": 3, + "lft": 52, + "tree_id": 90, + "bbox_x0": 24.698099, + "bbox_x1": 36.89468, + "bbox_y0": 22, + "bbox_y1": 31.674179 + } + }, + { + "pk": 81, + "model": "base.region", + "fields": { + "rght": 193, + "code": "SLV", + "name": "El Salvador", + "parent": 3, + "level": 3, + "lft": 192, + "tree_id": 90, + "bbox_x0": -90.12867, + "bbox_x1": -87.682869, + "bbox_y0": 13.14867, + "bbox_y1": 14.44506 + } + }, + { + "pk": 82, + "model": "base.region", + "fields": { + "rght": 97, + "code": "GNQ", + "name": "Equatorial Guinea", + "parent": 13, + "level": 3, + "lft": 96, + "tree_id": 90, + "bbox_x0": 5.60236, + "bbox_x1": 11.33572, + "bbox_y0": -1.48378, + "bbox_y1": 3.78597 + } + }, + { + "pk": 83, + "model": "base.region", + "fields": { + "rght": 21, + "code": "ERI", + "name": "Eritrea", + "parent": 12, + "level": 3, + "lft": 20, + "tree_id": 90, + "bbox_x0": 36.43877, + "bbox_x1": 43.14864, + "bbox_y0": 12.35956, + "bbox_y1": 18.00308 + } + }, + { + "pk": 84, + "model": "base.region", + "fields": { + "rght": 350, + "code": "EST", + "name": "Estonia", + "parent": 5, + "level": 2, + "lft": 349, + "tree_id": 90, + "bbox_x0": 21.771851, + "bbox_x1": 28.20989, + "bbox_y0": 57.509312, + "bbox_y1": 59.685749 + } + }, + { + "pk": 85, + "model": "base.region", + "fields": { + "rght": 23, + "code": "ETH", + "name": "Ethiopia", + "parent": 12, + "level": 3, + "lft": 22, + "tree_id": 90, + "bbox_x0": 32.99992, + "bbox_x1": 47.986172, + "bbox_y0": 3.40242, + "bbox_y1": 14.89218 + } + }, + { + "pk": 86, + "model": "base.region", + "fields": { + "rght": 352, + "code": "FRO", + "name": "Faeroe Islands", + "parent": 5, + "level": 2, + "lft": 351, + "tree_id": 90, + "bbox_x0": -7.68124, + "bbox_x1": -6.25861, + "bbox_y0": 61.394932, + "bbox_y1": 62.400742 + } + }, + { + "pk": 87, + "model": "base.region", + "fields": { + "rght": 227, + "code": "FLK", + "name": "Falkland Islands (Malvinas)", + "parent": 4, + "level": 3, + "lft": 226, + "tree_id": 90, + "bbox_x0": -61.43404, + "bbox_x1": -57.712479, + "bbox_y0": -52.900581, + "bbox_y1": -50.966221 + } + }, + { + "pk": 88, + "model": "base.region", + "fields": { + "rght": 472, + "code": "FJI", + "name": "Fiji", + "parent": 256, + "level": 2, + "lft": 471, + "tree_id": 90, + "bbox_x0": 174.866196, + "bbox_x1": -178.203156, + "bbox_y0": -21.01712, + "bbox_y1": -12.46622 + } + }, + { + "pk": 89, + "model": "base.region", + "fields": { + "rght": 354, + "code": "FIN", + "name": "Finland", + "parent": 5, + "level": 2, + "lft": 353, + "tree_id": 90, + "bbox_x0": 20.548571, + "bbox_x1": 31.586201, + "bbox_y0": 59.764881, + "bbox_y1": 70.092308 + } + }, + { + "pk": 90, + "model": "base.region", + "fields": { + "rght": 356, + "code": "FRA", + "name": "France", + "parent": 5, + "level": 2, + "lft": 355, + "tree_id": 90, + "bbox_x0": -5.1406, + "bbox_x1": 9.55932, + "bbox_y0": 41.33374, + "bbox_y1": 51.089062 + } + }, + { + "pk": 91, + "model": "base.region", + "fields": { + "rght": 229, + "code": "GUF", + "name": "French Guiana", + "parent": 4, + "level": 3, + "lft": 228, + "tree_id": 90, + "bbox_x0": -54.542511, + "bbox_x1": -51.613941, + "bbox_y0": 2.12709, + "bbox_y1": 5.77649 + } + }, + { + "pk": 92, + "model": "base.region", + "fields": { + "rght": 474, + "code": "PYF", + "name": "French Polynesia", + "parent": 256, + "level": 2, + "lft": 473, + "tree_id": 90, + "bbox_x0": -154.700485, + "bbox_x1": -108.87291, + "bbox_y0": -27.65357, + "bbox_y1": 10.35983 + } + }, + { + "pk": 93, + "model": "base.region", + "fields": { + "rght": 99, + "code": "GAB", + "name": "Gabon", + "parent": 13, + "level": 3, + "lft": 98, + "tree_id": 90, + "bbox_x0": 8.69547, + "bbox_x1": 14.50234, + "bbox_y0": -3.9788, + "bbox_y1": 2.32261 + } + }, + { + "pk": 94, + "model": "base.region", + "fields": { + "rght": 101, + "code": "GMB", + "name": "Gambia", + "parent": 13, + "level": 3, + "lft": 100, + "tree_id": 90, + "bbox_x0": -16.82506, + "bbox_x1": -13.7978, + "bbox_y0": 13.06425, + "bbox_y1": 13.82657 + } + }, + { + "pk": 95, + "model": "base.region", + "fields": { + "rght": 358, + "code": "GEO", + "name": "Georgia", + "parent": 5, + "level": 2, + "lft": 357, + "tree_id": 90, + "bbox_x0": 40.01022, + "bbox_x1": 46.721359, + "bbox_y0": 41.038502, + "bbox_y1": 43.584549 + } + }, + { + "pk": 96, + "model": "base.region", + "fields": { + "rght": 360, + "code": "DEU", + "name": "Germany", + "parent": 5, + "level": 2, + "lft": 359, + "tree_id": 90, + "bbox_x0": 5.86624, + "bbox_x1": 15.04205, + "bbox_y0": 47.27021, + "bbox_y1": 55.05814 + } + }, + { + "pk": 97, + "model": "base.region", + "fields": { + "rght": 103, + "code": "GHA", + "name": "Ghana", + "parent": 13, + "level": 3, + "lft": 102, + "tree_id": 90, + "bbox_x0": -3.25542, + "bbox_x1": 1.19178, + "bbox_y0": 4.73672, + "bbox_y1": 11.1733 + } + }, + { + "pk": 98, + "model": "base.region", + "fields": { + "rght": 362, + "code": "GIB", + "name": "Gibraltar", + "parent": 5, + "level": 2, + "lft": 361, + "tree_id": 90, + "bbox_x0": -5.3579, + "bbox_x1": -5.33867, + "bbox_y0": 36.108219, + "bbox_y1": 36.15593 + } + }, + { + "pk": 99, + "model": "base.region", + "fields": { + "rght": 364, + "code": "GRC", + "name": "Greece", + "parent": 5, + "level": 2, + "lft": 363, + "tree_id": 90, + "bbox_x0": 19.37431, + "bbox_x1": 29.70056, + "bbox_y0": 34.809502, + "bbox_y1": 41.757111 + } + }, + { + "pk": 100, + "model": "base.region", + "fields": { + "rght": 207, + "code": "GRL", + "name": "Greenland", + "parent": 2, + "level": 3, + "lft": 206, + "tree_id": 90, + "bbox_x0": -73.263474, + "bbox_x1": -11.31232, + "bbox_y0": 59.777271, + "bbox_y1": 83.627419 + } + }, + { + "pk": 101, + "model": "base.region", + "fields": { + "rght": 153, + "code": "GRD", + "name": "Grenada", + "parent": 255, + "level": 3, + "lft": 152, + "tree_id": 90, + "bbox_x0": -61.79998, + "bbox_x1": -61.376362, + "bbox_y0": 11.98288, + "bbox_y1": 12.5415 + } + }, + { + "pk": 102, + "model": "base.region", + "fields": { + "rght": 155, + "code": "GLP", + "name": "Guadeloupe", + "parent": 255, + "level": 3, + "lft": 154, + "tree_id": 90, + "bbox_x0": -61.807159, + "bbox_x1": -61, + "bbox_y0": 15.83097, + "bbox_y1": 16.51684 + } + }, + { + "pk": 103, + "model": "base.region", + "fields": { + "rght": 476, + "code": "GUM", + "name": "Guam", + "parent": 256, + "level": 2, + "lft": 475, + "tree_id": 90, + "bbox_x0": 144.619263, + "bbox_x1": 144.953995, + "bbox_y0": 13.24059, + "bbox_y1": 13.65232 + } + }, + { + "pk": 104, + "model": "base.region", + "fields": { + "rght": 195, + "code": "GTM", + "name": "Guatemala", + "parent": 3, + "level": 3, + "lft": 194, + "tree_id": 90, + "bbox_x0": -92.241432, + "bbox_x1": -88.22319, + "bbox_y0": 13.7373, + "bbox_y1": 17.815201 + } + }, + { + "pk": 105, + "model": "base.region", + "fields": { + "rght": 366, + "code": "GGY", + "name": "Guernsey", + "parent": 5, + "level": 2, + "lft": 365, + "tree_id": 90, + "bbox_x0": -2.67545, + "bbox_x1": -2.16382, + "bbox_y0": 49.405762, + "bbox_y1": 49.7393 + } + }, + { + "pk": 106, + "model": "base.region", + "fields": { + "rght": 105, + "code": "GIN", + "name": "Guinea", + "parent": 13, + "level": 3, + "lft": 104, + "tree_id": 90, + "bbox_x0": -15.08625, + "bbox_x1": -7.64106, + "bbox_y0": 7.19355, + "bbox_y1": 12.67621 + } + }, + { + "pk": 107, + "model": "base.region", + "fields": { + "rght": 107, + "code": "GNB", + "name": "Guinea-Bissau", + "parent": 13, + "level": 3, + "lft": 106, + "tree_id": 90, + "bbox_x0": -16.717529, + "bbox_x1": -13.63652, + "bbox_y0": 10.85997, + "bbox_y1": 12.68078 + } + }, + { + "pk": 108, + "model": "base.region", + "fields": { + "rght": 231, + "code": "GUY", + "name": "Guyana", + "parent": 4, + "level": 3, + "lft": 230, + "tree_id": 90, + "bbox_x0": -61.396271, + "bbox_x1": -56.480251, + "bbox_y0": 1.17508, + "bbox_y1": 8.55756 + } + }, + { + "pk": 109, + "model": "base.region", + "fields": { + "rght": 157, + "code": "HTI", + "name": "Haiti", + "parent": 255, + "level": 3, + "lft": 156, + "tree_id": 90, + "bbox_x0": -74.478592, + "bbox_x1": -71.61335, + "bbox_y0": 18.02103, + "bbox_y1": 20.08782 + } + }, + { + "pk": 110, + "model": "base.region", + "fields": { + "rght": 368, + "code": "VAT", + "name": "Holy See (Vatican City)", + "parent": 5, + "level": 2, + "lft": 367, + "tree_id": 90, + "bbox_x0": 12.44584, + "bbox_x1": 12.45842, + "bbox_y0": 41.900211, + "bbox_y1": 41.907459 + } + }, + { + "pk": 111, + "model": "base.region", + "fields": { + "rght": 197, + "code": "HND", + "name": "Honduras", + "parent": 3, + "level": 3, + "lft": 196, + "tree_id": 90, + "bbox_x0": -89.350792, + "bbox_x1": -82.499527, + "bbox_y0": 12.98241, + "bbox_y1": 17.450451 + } + }, + { + "pk": 112, + "model": "base.region", + "fields": { + "rght": 370, + "code": "HUN", + "name": "Hungary", + "parent": 5, + "level": 2, + "lft": 369, + "tree_id": 90, + "bbox_x0": 16.11335, + "bbox_x1": 22.89657, + "bbox_y0": 45.737061, + "bbox_y1": 48.585258 + } + }, + { + "pk": 113, + "model": "base.region", + "fields": { + "rght": 372, + "code": "ISL", + "name": "Iceland", + "parent": 5, + "level": 2, + "lft": 371, + "tree_id": 90, + "bbox_x0": -24.54652, + "bbox_x1": -13.49416, + "bbox_y0": 63.295952, + "bbox_y1": 66.566193 + } + }, + { + "pk": 114, + "model": "base.region", + "fields": { + "rght": 283, + "code": "IND", + "name": "India", + "parent": 9, + "level": 3, + "lft": 282, + "tree_id": 90, + "bbox_x0": 68.032318, + "bbox_x1": 97.403023, + "bbox_y0": 6.7471, + "bbox_y1": 36.261688 + } + }, + { + "pk": 115, + "model": "base.region", + "fields": { + "rght": 299, + "code": "IDN", + "name": "Indonesia", + "parent": 7, + "level": 3, + "lft": 298, + "tree_id": 90, + "bbox_x0": 94.969833, + "bbox_x1": 141.021805, + "bbox_y0": -11.00485, + "bbox_y1": 6.07573 + } + }, + { + "pk": 116, + "model": "base.region", + "fields": { + "rght": 438, + "code": "IRN", + "name": "Iran", + "parent": 15, + "level": 2, + "lft": 437, + "tree_id": 90, + "bbox_x0": 44.047249, + "bbox_x1": 63.317459, + "bbox_y0": 25.064079, + "bbox_y1": 39.777222 + } + }, + { + "pk": 117, + "model": "base.region", + "fields": { + "rght": 440, + "code": "IRQ", + "name": "Iraq", + "parent": 15, + "level": 2, + "lft": 439, + "tree_id": 90, + "bbox_x0": 38.804001, + "bbox_x1": 48.575699, + "bbox_y0": 29.103001, + "bbox_y1": 37.378052 + } + }, + { + "pk": 118, + "model": "base.region", + "fields": { + "rght": 374, + "code": "IRL", + "name": "Ireland", + "parent": 5, + "level": 2, + "lft": 373, + "tree_id": 90, + "bbox_x0": -10.61834, + "bbox_x1": -5.9975, + "bbox_y0": 51.424511, + "bbox_y1": 55.436211 + } + }, + { + "pk": 119, + "model": "base.region", + "fields": { + "rght": 376, + "code": "IMN", + "name": "Isle of Man", + "parent": 5, + "level": 2, + "lft": 375, + "tree_id": 90, + "bbox_x0": -4.83018, + "bbox_x1": -4.31007, + "bbox_y0": 54.04464, + "bbox_y1": 54.418839 + } + }, + { + "pk": 120, + "model": "base.region", + "fields": { + "rght": 442, + "code": "ISR", + "name": "Israel", + "parent": 15, + "level": 2, + "lft": 441, + "tree_id": 90, + "bbox_x0": 34.2677, + "bbox_x1": 35.940941, + "bbox_y0": 29.4965, + "bbox_y1": 33.43338 + } + }, + { + "pk": 121, + "model": "base.region", + "fields": { + "rght": 378, + "code": "ITA", + "name": "Italy", + "parent": 5, + "level": 2, + "lft": 377, + "tree_id": 90, + "bbox_x0": 6.62665, + "bbox_x1": 18.520281, + "bbox_y0": 35.49308, + "bbox_y1": 47.091999 + } + }, + { + "pk": 122, + "model": "base.region", + "fields": { + "rght": 159, + "code": "JAM", + "name": "Jamaica", + "parent": 255, + "level": 3, + "lft": 158, + "tree_id": 90, + "bbox_x0": -78.366638, + "bbox_x1": -75.982857, + "bbox_y0": 16.949551, + "bbox_y1": 18.52697 + } + }, + { + "pk": 123, + "model": "base.region", + "fields": { + "rght": 271, + "code": "JPN", + "name": "Japan", + "parent": 258, + "level": 3, + "lft": 270, + "tree_id": 90, + "bbox_x0": 122.933647, + "bbox_x1": 153.986847, + "bbox_y0": 20.4251, + "bbox_y1": 45.557709 + } + }, + { + "pk": 124, + "model": "base.region", + "fields": { + "rght": 380, + "code": "JEY", + "name": "Jersey", + "parent": 5, + "level": 2, + "lft": 379, + "tree_id": 90, + "bbox_x0": -2.25505, + "bbox_x1": -2.01129, + "bbox_y0": 49.16209, + "bbox_y1": 49.26231 + } + }, + { + "pk": 125, + "model": "base.region", + "fields": { + "rght": 444, + "code": "JOR", + "name": "Jordan", + "parent": 15, + "level": 2, + "lft": 443, + "tree_id": 90, + "bbox_x0": 34.960232, + "bbox_x1": 39.301128, + "bbox_y0": 29.18409, + "bbox_y1": 33.374828 + } + }, + { + "pk": 126, + "model": "base.region", + "fields": { + "rght": 251, + "code": "KAZ", + "name": "Kazakhstan", + "parent": 8, + "level": 3, + "lft": 250, + "tree_id": 90, + "bbox_x0": 46.491859, + "bbox_x1": 87.312737, + "bbox_y0": 40.566689, + "bbox_y1": 55.431808 + } + }, + { + "pk": 127, + "model": "base.region", + "fields": { + "rght": 25, + "code": "KEN", + "name": "Kenya", + "parent": 12, + "level": 3, + "lft": 24, + "tree_id": 90, + "bbox_x0": 33.90884, + "bbox_x1": 41.899059, + "bbox_y0": -4.71712, + "bbox_y1": 4.62933 + } + }, + { + "pk": 128, + "model": "base.region", + "fields": { + "rght": 478, + "code": "KIR", + "name": "Kiribati", + "parent": 256, + "level": 2, + "lft": 477, + "tree_id": 90, + "bbox_x0": 158.418335, + "bbox_x1": -150.208359, + "bbox_y0": -11.43703, + "bbox_y1": 4.71956 + } + }, + { + "pk": 129, + "model": "base.region", + "fields": { + "rght": 446, + "code": "KWT", + "name": "Kuwait", + "parent": 15, + "level": 2, + "lft": 445, + "tree_id": 90, + "bbox_x0": 46.55751, + "bbox_x1": 48.78384, + "bbox_y0": 28.5245, + "bbox_y1": 30.0958 + } + }, + { + "pk": 130, + "model": "base.region", + "fields": { + "rght": 253, + "code": "KGZ", + "name": "Kyrgyzstan", + "parent": 8, + "level": 3, + "lft": 252, + "tree_id": 90, + "bbox_x0": 69.276619, + "bbox_x1": 80.28318, + "bbox_y0": 39.17284, + "bbox_y1": 43.238239 + } + }, + { + "pk": 131, + "model": "base.region", + "fields": { + "rght": 301, + "code": "LAO", + "name": "Lao People's Democratic Republic", + "parent": 7, + "level": 3, + "lft": 300, + "tree_id": 90, + "bbox_x0": 100.093048, + "bbox_x1": 107.697021, + "bbox_y0": 13.91002, + "bbox_y1": 22.500389 + } + }, + { + "pk": 132, + "model": "base.region", + "fields": { + "rght": 382, + "code": "LVA", + "name": "Latvia", + "parent": 5, + "level": 2, + "lft": 381, + "tree_id": 90, + "bbox_x0": 20.966061, + "bbox_x1": 28.244431, + "bbox_y0": 55.67276, + "bbox_y1": 58.087448 + } + }, + { + "pk": 133, + "model": "base.region", + "fields": { + "rght": 448, + "code": "LBN", + "name": "Lebanon", + "parent": 15, + "level": 2, + "lft": 447, + "tree_id": 90, + "bbox_x0": 35.10368, + "bbox_x1": 36.622791, + "bbox_y0": 33.048908, + "bbox_y1": 34.69268 + } + }, + { + "pk": 134, + "model": "base.region", + "fields": { + "rght": 69, + "code": "LSO", + "name": "Lesotho", + "parent": 14, + "level": 3, + "lft": 68, + "tree_id": 90, + "bbox_x0": 27.02906, + "bbox_x1": 29.465771, + "bbox_y0": -30.668961, + "bbox_y1": -28.57205 + } + }, + { + "pk": 135, + "model": "base.region", + "fields": { + "rght": 109, + "code": "LBR", + "name": "Liberia", + "parent": 13, + "level": 3, + "lft": 108, + "tree_id": 90, + "bbox_x0": -11.49208, + "bbox_x1": -7.36511, + "bbox_y0": 4.35305, + "bbox_y1": 8.55179 + } + }, + { + "pk": 136, + "model": "base.region", + "fields": { + "rght": 55, + "code": "LBY", + "name": "Libyan Arab Jamahiriya", + "parent": 11, + "level": 3, + "lft": 54, + "tree_id": 90, + "bbox_x0": 9.38702, + "bbox_x1": 25.15061, + "bbox_y0": 19.508039, + "bbox_y1": 33.168999 + } + }, + { + "pk": 137, + "model": "base.region", + "fields": { + "rght": 384, + "code": "LIE", + "name": "Liechtenstein", + "parent": 5, + "level": 2, + "lft": 383, + "tree_id": 90, + "bbox_x0": 9.47181, + "bbox_x1": 9.63578, + "bbox_y0": 47.04834, + "bbox_y1": 47.270649 + } + }, + { + "pk": 138, + "model": "base.region", + "fields": { + "rght": 386, + "code": "LTU", + "name": "Lithuania", + "parent": 5, + "level": 2, + "lft": 385, + "tree_id": 90, + "bbox_x0": 20.953199, + "bbox_x1": 26.835581, + "bbox_y0": 53.89748, + "bbox_y1": 56.450432 + } + }, + { + "pk": 139, + "model": "base.region", + "fields": { + "rght": 388, + "code": "LUX", + "name": "Luxembourg", + "parent": 5, + "level": 2, + "lft": 387, + "tree_id": 90, + "bbox_x0": 5.73579, + "bbox_x1": 6.53117, + "bbox_y0": 49.447689, + "bbox_y1": 50.182751 + } + }, + { + "pk": 140, + "model": "base.region", + "fields": { + "rght": 390, + "code": "MKD", + "name": "Macedonia", + "parent": 5, + "level": 2, + "lft": 389, + "tree_id": 90, + "bbox_x0": 20.4645, + "bbox_x1": 23.038071, + "bbox_y0": 40.860111, + "bbox_y1": 42.345501 + } + }, + { + "pk": 141, + "model": "base.region", + "fields": { + "rght": 27, + "code": "MDG", + "name": "Madagascar", + "parent": 12, + "level": 3, + "lft": 26, + "tree_id": 90, + "bbox_x0": 43.19138, + "bbox_x1": 50.483799, + "bbox_y0": -25.60894, + "bbox_y1": -11.94543 + } + }, + { + "pk": 142, + "model": "base.region", + "fields": { + "rght": 29, + "code": "MWI", + "name": "Malawi", + "parent": 12, + "level": 3, + "lft": 28, + "tree_id": 90, + "bbox_x0": 32.668991, + "bbox_x1": 35.920441, + "bbox_y0": -17.129459, + "bbox_y1": -9.36468 + } + }, + { + "pk": 143, + "model": "base.region", + "fields": { + "rght": 303, + "code": "MYS", + "name": "Malaysia", + "parent": 7, + "level": 3, + "lft": 302, + "tree_id": 90, + "bbox_x0": 98.935059, + "bbox_x1": 119.448433, + "bbox_y0": 0.66364, + "bbox_y1": 7.58378 + } + }, + { + "pk": 144, + "model": "base.region", + "fields": { + "rght": 285, + "code": "MDV", + "name": "Maldives", + "parent": 9, + "level": 3, + "lft": 284, + "tree_id": 90, + "bbox_x0": 72.616219, + "bbox_x1": 73.76712, + "bbox_y0": -2.90045, + "bbox_y1": 7.11712 + } + }, + { + "pk": 145, + "model": "base.region", + "fields": { + "rght": 111, + "code": "MLI", + "name": "Mali", + "parent": 13, + "level": 3, + "lft": 110, + "tree_id": 90, + "bbox_x0": -12.24261, + "bbox_x1": 4.24495, + "bbox_y0": 10.15951, + "bbox_y1": 25 + } + }, + { + "pk": 146, + "model": "base.region", + "fields": { + "rght": 392, + "code": "MLT", + "name": "Malta", + "parent": 5, + "level": 2, + "lft": 391, + "tree_id": 90, + "bbox_x0": 14.18341, + "bbox_x1": 14.5766, + "bbox_y0": 35.786282, + "bbox_y1": 36.081821 + } + }, + { + "pk": 147, + "model": "base.region", + "fields": { + "rght": 480, + "code": "MHL", + "name": "Marshall Islands", + "parent": 256, + "level": 2, + "lft": 479, + "tree_id": 90, + "bbox_x0": 162.143265, + "bbox_x1": 172.161987, + "bbox_y0": 4.57487, + "bbox_y1": 14.65516 + } + }, + { + "pk": 148, + "model": "base.region", + "fields": { + "rght": 161, + "code": "MTQ", + "name": "Martinique", + "parent": 255, + "level": 3, + "lft": 160, + "tree_id": 90, + "bbox_x0": -61.23011, + "bbox_x1": -60.81551, + "bbox_y0": 14.38244, + "bbox_y1": 14.87881 + } + }, + { + "pk": 149, + "model": "base.region", + "fields": { + "rght": 113, + "code": "MRT", + "name": "Mauritania", + "parent": 13, + "level": 3, + "lft": 112, + "tree_id": 90, + "bbox_x0": -17.066521, + "bbox_x1": -4.8352, + "bbox_y0": 14.71554, + "bbox_y1": 27.298071 + } + }, + { + "pk": 150, + "model": "base.region", + "fields": { + "rght": 31, + "code": "MUS", + "name": "Mauritius", + "parent": 12, + "level": 3, + "lft": 30, + "tree_id": 90, + "bbox_x0": 56.512711, + "bbox_x1": 63.525379, + "bbox_y0": -20.525709, + "bbox_y1": -10.31925 + } + }, + { + "pk": 151, + "model": "base.region", + "fields": { + "rght": 33, + "code": "MYT", + "name": "Mayotte", + "parent": 12, + "level": 3, + "lft": 32, + "tree_id": 90, + "bbox_x0": 45.01461, + "bbox_x1": 45.317131, + "bbox_y0": -13.00045, + "bbox_y1": -12.63383 + } + }, + { + "pk": 152, + "model": "base.region", + "fields": { + "rght": 209, + "code": "MEX", + "name": "Mexico", + "parent": 2, + "level": 3, + "lft": 208, + "tree_id": 90, + "bbox_x0": -118.867172, + "bbox_x1": -86.703392, + "bbox_y0": 14.53285, + "bbox_y1": 32.71862 + } + }, + { + "pk": 153, + "model": "base.region", + "fields": { + "rght": 482, + "code": "FSM", + "name": "Micronesia, Federated States of", + "parent": 256, + "level": 2, + "lft": 481, + "tree_id": 90, + "bbox_x0": 138.052856, + "bbox_x1": 163.034912, + "bbox_y0": 5.25984, + "bbox_y1": 10.02222 + } + }, + { + "pk": 154, + "model": "base.region", + "fields": { + "rght": 394, + "code": "MCO", + "name": "Monaco", + "parent": 5, + "level": 2, + "lft": 393, + "tree_id": 90, + "bbox_x0": 7.4091, + "bbox_x1": 7.43948, + "bbox_y0": 43.724789, + "bbox_y1": 43.7519 + } + }, + { + "pk": 155, + "model": "base.region", + "fields": { + "rght": 273, + "code": "MNG", + "name": "Mongolia", + "parent": 258, + "level": 3, + "lft": 272, + "tree_id": 90, + "bbox_x0": 87.749496, + "bbox_x1": 119.92411, + "bbox_y0": 41.567501, + "bbox_y1": 52.154099 + } + }, + { + "pk": 156, + "model": "base.region", + "fields": { + "rght": 396, + "code": "MNE", + "name": "Montenegro", + "parent": 5, + "level": 2, + "lft": 395, + "tree_id": 90, + "bbox_x0": 18.43705, + "bbox_x1": 20.41608, + "bbox_y0": 41.84808, + "bbox_y1": 43.542912 + } + }, + { + "pk": 157, + "model": "base.region", + "fields": { + "rght": 163, + "code": "MSR", + "name": "Montserrat", + "parent": 255, + "level": 3, + "lft": 162, + "tree_id": 90, + "bbox_x0": -62.24258, + "bbox_x1": -62.14642, + "bbox_y0": 16.671, + "bbox_y1": 16.81732 + } + }, + { + "pk": 158, + "model": "base.region", + "fields": { + "rght": 57, + "code": "MAR", + "name": "Morocco", + "parent": 11, + "level": 3, + "lft": 56, + "tree_id": 90, + "bbox_x0": -11.7805, + "bbox_x1": -1.02441, + "bbox_y0": 26.946489, + "bbox_y1": 35.921909 + } + }, + { + "pk": 159, + "model": "base.region", + "fields": { + "rght": 35, + "code": "MOZ", + "name": "Mozambique", + "parent": 12, + "level": 3, + "lft": 34, + "tree_id": 90, + "bbox_x0": 30.21731, + "bbox_x1": 40.844471, + "bbox_y0": -26.868679, + "bbox_y1": -10.47188 + } + }, + { + "pk": 160, + "model": "base.region", + "fields": { + "rght": 305, + "code": "MMR", + "name": "Myanmar", + "parent": 7, + "level": 3, + "lft": 304, + "tree_id": 90, + "bbox_x0": 92.189209, + "bbox_x1": 101.176788, + "bbox_y0": 9.60035, + "bbox_y1": 28.543249 + } + }, + { + "pk": 161, + "model": "base.region", + "fields": { + "rght": 71, + "code": "NMB", + "name": "Namibia", + "parent": 14, + "level": 3, + "lft": 70, + "tree_id": 90, + "bbox_x0": 11.71563, + "bbox_x1": 25.256701, + "bbox_y0": -28.97142, + "bbox_y1": -16.95989 + } + }, + { + "pk": 162, + "model": "base.region", + "fields": { + "rght": 484, + "code": "NRU", + "name": "Nauru", + "parent": 256, + "level": 2, + "lft": 483, + "tree_id": 90, + "bbox_x0": 166.899017, + "bbox_x1": 166.945267, + "bbox_y0": -0.55232, + "bbox_y1": -0.50429 + } + }, + { + "pk": 163, + "model": "base.region", + "fields": { + "rght": 287, + "code": "NPL", + "name": "Nepal", + "parent": 9, + "level": 3, + "lft": 286, + "tree_id": 90, + "bbox_x0": 80.056221, + "bbox_x1": 88.199318, + "bbox_y0": 26.356501, + "bbox_y1": 30.433001 + } + }, + { + "pk": 164, + "model": "base.region", + "fields": { + "rght": 398, + "code": "NLD", + "name": "Netherlands", + "parent": 5, + "level": 2, + "lft": 397, + "tree_id": 90, + "bbox_x0": 3.35794, + "bbox_x1": 7.2267, + "bbox_y0": 50.750401, + "bbox_y1": 53.554241 + } + }, + { + "pk": 165, + "model": "base.region", + "fields": { + "rght": 165, + "code": "NAN", + "name": "Netherlands Antilles", + "parent": 255, + "level": 3, + "lft": 164, + "tree_id": 90, + "bbox_x0": -69.157188, + "bbox_x1": -62.943661, + "bbox_y0": 11.97318, + "bbox_y1": 18.07024 + } + }, + { + "pk": 166, + "model": "base.region", + "fields": { + "rght": 486, + "code": "NCL", + "name": "New Caledonia", + "parent": 256, + "level": 2, + "lft": 485, + "tree_id": 90, + "bbox_x0": 158.332855, + "bbox_x1": 172.061417, + "bbox_y0": -22.90045, + "bbox_y1": -18.01622 + } + }, + { + "pk": 167, + "model": "base.region", + "fields": { + "rght": 488, + "code": "NZL", + "name": "New Zealand", + "parent": 256, + "level": 2, + "lft": 487, + "tree_id": 90, + "bbox_x0": 165.883804, + "bbox_x1": -175.987198, + "bbox_y0": -52.618591, + "bbox_y1": -29.20997 + } + }, + { + "pk": 168, + "model": "base.region", + "fields": { + "rght": 199, + "code": "NIC", + "name": "Nicaragua", + "parent": 3, + "level": 3, + "lft": 198, + "tree_id": 90, + "bbox_x0": -87.6903, + "bbox_x1": -82.592072, + "bbox_y0": 10.70754, + "bbox_y1": 15.0259 + } + }, + { + "pk": 169, + "model": "base.region", + "fields": { + "rght": 115, + "code": "NER", + "name": "Niger", + "parent": 13, + "level": 3, + "lft": 114, + "tree_id": 90, + "bbox_x0": 0.16625, + "bbox_x1": 15.99564, + "bbox_y0": 11.69697, + "bbox_y1": 23.525021 + } + }, + { + "pk": 170, + "model": "base.region", + "fields": { + "rght": 117, + "code": "NGA", + "name": "Nigeria", + "parent": 13, + "level": 3, + "lft": 116, + "tree_id": 90, + "bbox_x0": 2.66844, + "bbox_x1": 14.68006, + "bbox_y0": 4.27714, + "bbox_y1": 13.892 + } + }, + { + "pk": 171, + "model": "base.region", + "fields": { + "rght": 490, + "code": "NIU", + "name": "Niue", + "parent": 256, + "level": 2, + "lft": 489, + "tree_id": 90, + "bbox_x0": -169.951004, + "bbox_x1": -169.775177, + "bbox_y0": -19.152189, + "bbox_y1": -18.951059 + } + }, + { + "pk": 172, + "model": "base.region", + "fields": { + "rght": 492, + "code": "NFK", + "name": "Norfolk Island", + "parent": 256, + "level": 2, + "lft": 491, + "tree_id": 90, + "bbox_x0": 167.949493, + "bbox_x1": 168.091614, + "bbox_y0": -29.11937, + "bbox_y1": -28.99493 + } + }, + { + "pk": 173, + "model": "base.region", + "fields": { + "rght": 494, + "code": "MNP", + "name": "Northern Mariana Islands", + "parent": 256, + "level": 2, + "lft": 493, + "tree_id": 90, + "bbox_x0": 136.082855, + "bbox_x1": 146.081223, + "bbox_y0": 14.10735, + "bbox_y1": 20.41712 + } + }, + { + "pk": 174, + "model": "base.region", + "fields": { + "rght": 400, + "code": "NOR", + "name": "Norway", + "parent": 5, + "level": 2, + "lft": 399, + "tree_id": 90, + "bbox_x0": 4.43292, + "bbox_x1": 31.168409, + "bbox_y0": 57.962582, + "bbox_y1": 71.185509 + } + }, + { + "pk": 175, + "model": "base.region", + "fields": { + "rght": 450, + "code": "PSE", + "name": "Occupied Palestinian Territory", + "parent": 15, + "level": 2, + "lft": 449, + "tree_id": 90, + "bbox_x0": 34.230461, + "bbox_x1": 35.876499, + "bbox_y0": 31.223499, + "bbox_y1": 33.340099 + } + }, + { + "pk": 176, + "model": "base.region", + "fields": { + "rght": 452, + "code": "OMN", + "name": "Oman", + "parent": 15, + "level": 2, + "lft": 451, + "tree_id": 90, + "bbox_x0": 51.882011, + "bbox_x1": 59.83651, + "bbox_y0": 16.6457, + "bbox_y1": 26.50045 + } + }, + { + "pk": 177, + "model": "base.region", + "fields": { + "rght": 289, + "code": "PAK", + "name": "Pakistan", + "parent": 9, + "level": 3, + "lft": 288, + "tree_id": 90, + "bbox_x0": 60.87859, + "bbox_x1": 77.840813, + "bbox_y0": 23.786699, + "bbox_y1": 37.097 + } + }, + { + "pk": 178, + "model": "base.region", + "fields": { + "rght": 496, + "code": "PLW", + "name": "Palau", + "parent": 256, + "level": 2, + "lft": 495, + "tree_id": 90, + "bbox_x0": 131.169235, + "bbox_x1": 134.723724, + "bbox_y0": 3.00114, + "bbox_y1": 8.09291 + } + }, + { + "pk": 179, + "model": "base.region", + "fields": { + "rght": 201, + "code": "PAN", + "name": "Panama", + "parent": 3, + "level": 3, + "lft": 200, + "tree_id": 90, + "bbox_x0": -83.051453, + "bbox_x1": -77.17411, + "bbox_y0": 7.1979, + "bbox_y1": 9.65045 + } + }, + { + "pk": 180, + "model": "base.region", + "fields": { + "rght": 498, + "code": "PNG", + "name": "Papua New Guinea", + "parent": 256, + "level": 2, + "lft": 497, + "tree_id": 90, + "bbox_x0": 140.842865, + "bbox_x1": 159.48378, + "bbox_y0": -11.65785, + "bbox_y1": -0.86622 + } + }, + { + "pk": 181, + "model": "base.region", + "fields": { + "rght": 233, + "code": "PRY", + "name": "Paraguay", + "parent": 4, + "level": 3, + "lft": 232, + "tree_id": 90, + "bbox_x0": -62.647072, + "bbox_x1": -54.25935, + "bbox_y0": -27.60873, + "bbox_y1": -19.294041 + } + }, + { + "pk": 182, + "model": "base.region", + "fields": { + "rght": 235, + "code": "PER", + "name": "Peru", + "parent": 4, + "level": 3, + "lft": 234, + "tree_id": 90, + "bbox_x0": -81.326736, + "bbox_x1": -68.677979, + "bbox_y0": -18.34972, + "bbox_y1": -0.01297 + } + }, + { + "pk": 183, + "model": "base.region", + "fields": { + "rght": 307, + "code": "PHL", + "name": "Philippines", + "parent": 7, + "level": 3, + "lft": 306, + "tree_id": 90, + "bbox_x0": 116.812721, + "bbox_x1": 126.856628, + "bbox_y0": 4.46811, + "bbox_y1": 21.23415 + } + }, + { + "pk": 184, + "model": "base.region", + "fields": { + "rght": 500, + "code": "PCN", + "name": "Pitcairn", + "parent": 256, + "level": 2, + "lft": 499, + "tree_id": 90, + "bbox_x0": -130.746033, + "bbox_x1": -124.772842, + "bbox_y0": -25.07752, + "bbox_y1": -23.917271 + } + }, + { + "pk": 185, + "model": "base.region", + "fields": { + "rght": 402, + "code": "POL", + "name": "Poland", + "parent": 5, + "level": 2, + "lft": 401, + "tree_id": 90, + "bbox_x0": 14.12281, + "bbox_x1": 24.145781, + "bbox_y0": 49.002022, + "bbox_y1": 54.835812 + } + }, + { + "pk": 186, + "model": "base.region", + "fields": { + "rght": 404, + "code": "PRT", + "name": "Portugal", + "parent": 5, + "level": 2, + "lft": 403, + "tree_id": 90, + "bbox_x0": -31.266001, + "bbox_x1": -6.18931, + "bbox_y0": 30.028061, + "bbox_y1": 42.154121 + } + }, + { + "pk": 187, + "model": "base.region", + "fields": { + "rght": 167, + "code": "PRI", + "name": "Puerto Rico", + "parent": 255, + "level": 3, + "lft": 166, + "tree_id": 90, + "bbox_x0": -67.942719, + "bbox_x1": -65.219978, + "bbox_y0": 17.883039, + "bbox_y1": 18.520161 + } + }, + { + "pk": 188, + "model": "base.region", + "fields": { + "rght": 454, + "code": "QAT", + "name": "Qatar", + "parent": 15, + "level": 2, + "lft": 453, + "tree_id": 90, + "bbox_x0": 50.757011, + "bbox_x1": 52.427509, + "bbox_y0": 24.482901, + "bbox_y1": 26.177509 + } + }, + { + "pk": 189, + "model": "base.region", + "fields": { + "rght": 275, + "code": "KOR", + "name": "Republic of Korea", + "parent": 258, + "level": 3, + "lft": 274, + "tree_id": 90, + "bbox_x0": 124.608147, + "bbox_x1": 130.933899, + "bbox_y0": 33.10611, + "bbox_y1": 38.612301 + } + }, + { + "pk": 190, + "model": "base.region", + "fields": { + "rght": 406, + "code": "MDA", + "name": "Republic of Moldova", + "parent": 5, + "level": 2, + "lft": 405, + "tree_id": 90, + "bbox_x0": 26.618879, + "bbox_x1": 30.16374, + "bbox_y0": 45.468498, + "bbox_y1": 48.490162 + } + }, + { + "pk": 191, + "model": "base.region", + "fields": { + "rght": 37, + "code": "REU", + "name": "Reunion", + "parent": 12, + "level": 3, + "lft": 36, + "tree_id": 90, + "bbox_x0": 55.219082, + "bbox_x1": 55.845039, + "bbox_y0": -21.37221, + "bbox_y1": -20.85685 + } + }, + { + "pk": 192, + "model": "base.region", + "fields": { + "rght": 408, + "code": "ROU", + "name": "Romania", + "parent": 5, + "level": 2, + "lft": 407, + "tree_id": 90, + "bbox_x0": 20.269791, + "bbox_x1": 29.691, + "bbox_y0": 43.626999, + "bbox_y1": 48.266891 + } + }, + { + "pk": 193, + "model": "base.region", + "fields": { + "rght": 410, + "code": "RUS", + "name": "Russian Federation", + "parent": 5, + "level": 2, + "lft": 409, + "tree_id": 90, + "bbox_x0": 19.638861, + "bbox_x1": 180, + "bbox_y0": 41.185902, + "bbox_y1": 81.856903 + } + }, + { + "pk": 194, + "model": "base.region", + "fields": { + "rght": 39, + "code": "RWA", + "name": "Rwanda", + "parent": 12, + "level": 3, + "lft": 38, + "tree_id": 90, + "bbox_x0": 28.8568, + "bbox_x1": 30.89596, + "bbox_y0": -2.84067, + "bbox_y1": -1.05348 + } + }, + { + "pk": 195, + "model": "base.region", + "fields": { + "rght": 73, + "code": "SHN", + "name": "Saint Helena", + "parent": 14, + "level": 3, + "lft": 72, + "tree_id": 90, + "bbox_x0": -14.44153, + "bbox_x1": -5.63286, + "bbox_y0": -40.400452, + "bbox_y1": -7.87757 + } + }, + { + "pk": 196, + "model": "base.region", + "fields": { + "rght": 169, + "code": "KNA", + "name": "Saint Kitts and Nevis", + "parent": 255, + "level": 3, + "lft": 168, + "tree_id": 90, + "bbox_x0": -62.86956, + "bbox_x1": -62.543259, + "bbox_y0": 17.095341, + "bbox_y1": 17.420111 + } + }, + { + "pk": 197, + "model": "base.region", + "fields": { + "rght": 171, + "code": "LCA", + "name": "Saint Lucia", + "parent": 255, + "level": 3, + "lft": 170, + "tree_id": 90, + "bbox_x0": -61.07415, + "bbox_x1": -60.866051, + "bbox_y0": 13.70477, + "bbox_y1": 14.10324 + } + }, + { + "pk": 198, + "model": "base.region", + "fields": { + "rght": 173, + "code": "SPM", + "name": "Saint Pierre and Miquelon", + "parent": 255, + "level": 3, + "lft": 172, + "tree_id": 90, + "bbox_x0": -56.420658, + "bbox_x1": -56.157372, + "bbox_y0": 46.753269, + "bbox_y1": 47.14629 + } + }, + { + "pk": 199, + "model": "base.region", + "fields": { + "rght": 175, + "code": "VCT", + "name": "Saint Vincent and the Grenadines", + "parent": 255, + "level": 3, + "lft": 174, + "tree_id": 90, + "bbox_x0": -61.459251, + "bbox_x1": -61.11388, + "bbox_y0": 12.58101, + "bbox_y1": 13.37783 + } + }, + { + "pk": 200, + "model": "base.region", + "fields": { + "rght": 177, + "code": "BLM", + "name": "Saint-Barthelemy", + "parent": 255, + "level": 3, + "lft": 176, + "tree_id": 90, + "bbox_x0": -62.9338, + "bbox_x1": -62.78286, + "bbox_y0": 17.86622, + "bbox_y1": 17.968599 + } + }, + { + "pk": 201, + "model": "base.region", + "fields": { + "rght": 179, + "code": "MAF", + "name": "Saint-Martin (French part)", + "parent": 255, + "level": 3, + "lft": 178, + "tree_id": 90, + "bbox_x0": -63.15276, + "bbox_x1": -62.972778, + "bbox_y0": 18.052231, + "bbox_y1": 18.13069 + } + }, + { + "pk": 202, + "model": "base.region", + "fields": { + "rght": 502, + "code": "WSM", + "name": "Samoa", + "parent": 256, + "level": 2, + "lft": 501, + "tree_id": 90, + "bbox_x0": -172.798584, + "bbox_x1": -171.33287, + "bbox_y0": -14.06353, + "bbox_y1": -13.4322 + } + }, + { + "pk": 203, + "model": "base.region", + "fields": { + "rght": 412, + "code": "SMR", + "name": "San Marino", + "parent": 5, + "level": 2, + "lft": 411, + "tree_id": 90, + "bbox_x0": 12.40306, + "bbox_x1": 12.51651, + "bbox_y0": 43.893299, + "bbox_y1": 43.992168 + } + }, + { + "pk": 204, + "model": "base.region", + "fields": { + "rght": 119, + "code": "STP", + "name": "Sao Tome and Principe", + "parent": 13, + "level": 3, + "lft": 118, + "tree_id": 90, + "bbox_x0": 5.59955, + "bbox_x1": 7.46637, + "bbox_y0": -0.014, + "bbox_y1": 1.73378 + } + }, + { + "pk": 205, + "model": "base.region", + "fields": { + "rght": 456, + "code": "SAU", + "name": "Saudi Arabia", + "parent": 15, + "level": 2, + "lft": 455, + "tree_id": 90, + "bbox_x0": 34.508282, + "bbox_x1": 55.666672, + "bbox_y0": 16.261169, + "bbox_y1": 32.173481 + } + }, + { + "pk": 206, + "model": "base.region", + "fields": { + "rght": 121, + "code": "SEN", + "name": "Senegal", + "parent": 13, + "level": 3, + "lft": 120, + "tree_id": 90, + "bbox_x0": -17.535231, + "bbox_x1": -11.35588, + "bbox_y0": 12.30727, + "bbox_y1": 16.691629 + } + }, + { + "pk": 207, + "model": "base.region", + "fields": { + "rght": 414, + "code": "SRB", + "name": "Serbia", + "parent": 5, + "level": 2, + "lft": 413, + "tree_id": 90, + "bbox_x0": 18.814581, + "bbox_x1": 23.007, + "bbox_y0": 41.908611, + "bbox_y1": 46.191002 + } + }, + { + "pk": 208, + "model": "base.region", + "fields": { + "rght": 41, + "code": "SYC", + "name": "Seychelles", + "parent": 12, + "level": 3, + "lft": 40, + "tree_id": 90, + "bbox_x0": 46.199211, + "bbox_x1": 56.279499, + "bbox_y0": -10.21712, + "bbox_y1": -3.71151 + } + }, + { + "pk": 209, + "model": "base.region", + "fields": { + "rght": 123, + "code": "SLE", + "name": "Sierra Leone", + "parent": 13, + "level": 3, + "lft": 122, + "tree_id": 90, + "bbox_x0": -13.30763, + "bbox_x1": -10.28423, + "bbox_y0": 6.92868, + "bbox_y1": 10.00043 + } + }, + { + "pk": 210, + "model": "base.region", + "fields": { + "rght": 309, + "code": "SGP", + "name": "Singapore", + "parent": 7, + "level": 3, + "lft": 308, + "tree_id": 90, + "bbox_x0": 103.618248, + "bbox_x1": 104.40847, + "bbox_y0": 1.1158, + "bbox_y1": 1.47062 + } + }, + { + "pk": 211, + "model": "base.region", + "fields": { + "rght": 416, + "code": "SVK", + "name": "Slovakia", + "parent": 5, + "level": 2, + "lft": 415, + "tree_id": 90, + "bbox_x0": 16.833179, + "bbox_x1": 22.570299, + "bbox_y0": 47.728001, + "bbox_y1": 49.603001 + } + }, + { + "pk": 212, + "model": "base.region", + "fields": { + "rght": 418, + "code": "SVN", + "name": "Slovenia", + "parent": 5, + "level": 2, + "lft": 417, + "tree_id": 90, + "bbox_x0": 13.37551, + "bbox_x1": 16.59656, + "bbox_y0": 45.416, + "bbox_y1": 46.87788 + } + }, + { + "pk": 213, + "model": "base.region", + "fields": { + "rght": 504, + "code": "SLB", + "name": "Solomon Islands", + "parent": 256, + "level": 2, + "lft": 503, + "tree_id": 90, + "bbox_x0": 155.508667, + "bbox_x1": 170.200455, + "bbox_y0": -12.2919, + "bbox_y1": -5.16622 + } + }, + { + "pk": 214, + "model": "base.region", + "fields": { + "rght": 43, + "code": "SOM", + "name": "Somalia", + "parent": 12, + "level": 3, + "lft": 42, + "tree_id": 90, + "bbox_x0": 40.988628, + "bbox_x1": 51.413029, + "bbox_y0": -1.66205, + "bbox_y1": 11.9852 + } + }, + { + "pk": 215, + "model": "base.region", + "fields": { + "rght": 75, + "code": "ZAF", + "name": "South Africa", + "parent": 14, + "level": 3, + "lft": 74, + "tree_id": 90, + "bbox_x0": 16.46841, + "bbox_x1": 37.993172, + "bbox_y0": -46.990009, + "bbox_y1": -22.12472 + } + }, + { + "pk": 216, + "model": "base.region", + "fields": { + "rght": 420, + "code": "ESP", + "name": "Spain", + "parent": 5, + "level": 2, + "lft": 419, + "tree_id": 90, + "bbox_x0": -18.160789, + "bbox_x1": 4.32788, + "bbox_y0": 27.63546, + "bbox_y1": 43.789959 + } + }, + { + "pk": 217, + "model": "base.region", + "fields": { + "rght": 291, + "code": "LKA", + "name": "Sri Lanka", + "parent": 9, + "level": 3, + "lft": 290, + "tree_id": 90, + "bbox_x0": 79.516212, + "bbox_x1": 81.88121, + "bbox_y0": 5.9167, + "bbox_y1": 9.8312 + } + }, + { + "pk": 218, + "model": "base.region", + "fields": { + "rght": 59, + "code": "SDN", + "name": "Sudan", + "parent": 11, + "level": 3, + "lft": 58, + "tree_id": 90, + "bbox_x0": 21.83894, + "bbox_x1": 38.833801, + "bbox_y0": 3.48639, + "bbox_y1": 23.146891 + } + }, + { + "pk": 219, + "model": "base.region", + "fields": { + "rght": 237, + "code": "SUR", + "name": "Suriname", + "parent": 4, + "level": 3, + "lft": 236, + "tree_id": 90, + "bbox_x0": -58.086559, + "bbox_x1": -53.977489, + "bbox_y0": 1.83114, + "bbox_y1": 6.00454 + } + }, + { + "pk": 220, + "model": "base.region", + "fields": { + "rght": 422, + "code": "SJM", + "name": "Svalbard and Jan Mayen Islands", + "parent": 5, + "level": 2, + "lft": 421, + "tree_id": 90, + "bbox_x0": -9.07989, + "bbox_x1": 36.815269, + "bbox_y0": 70.82737, + "bbox_y1": 80.834061 + } + }, + { + "pk": 221, + "model": "base.region", + "fields": { + "rght": 77, + "code": "SWZ", + "name": "Swaziland", + "parent": 14, + "level": 3, + "lft": 76, + "tree_id": 90, + "bbox_x0": 30.7941, + "bbox_x1": 32.137272, + "bbox_y0": -27.317101, + "bbox_y1": -25.719641 + } + }, + { + "pk": 222, + "model": "base.region", + "fields": { + "rght": 424, + "code": "SWE", + "name": "Sweden", + "parent": 5, + "level": 2, + "lft": 423, + "tree_id": 90, + "bbox_x0": 10.9661, + "bbox_x1": 24.16634, + "bbox_y0": 55.33696, + "bbox_y1": 69.059937 + } + }, + { + "pk": 223, + "model": "base.region", + "fields": { + "rght": 426, + "code": "CHE", + "name": "Switzerland", + "parent": 5, + "level": 2, + "lft": 425, + "tree_id": 90, + "bbox_x0": 5.95587, + "bbox_x1": 10.49203, + "bbox_y0": 45.81802, + "bbox_y1": 47.80838 + } + }, + { + "pk": 224, + "model": "base.region", + "fields": { + "rght": 458, + "code": "SYR", + "name": "Syrian Arab Republic", + "parent": 15, + "level": 2, + "lft": 457, + "tree_id": 90, + "bbox_x0": 35.727001, + "bbox_x1": 42.384998, + "bbox_y0": 32.3106, + "bbox_y1": 37.319 + } + }, + { + "pk": 225, + "model": "base.region", + "fields": { + "rght": 255, + "code": "TJK", + "name": "Tajikistan", + "parent": 8, + "level": 3, + "lft": 254, + "tree_id": 90, + "bbox_x0": 67.387131, + "bbox_x1": 75.137222, + "bbox_y0": 36.674141, + "bbox_y1": 41.04224 + } + }, + { + "pk": 226, + "model": "base.region", + "fields": { + "rght": 311, + "code": "THA", + "name": "Thailand", + "parent": 7, + "level": 3, + "lft": 310, + "tree_id": 90, + "bbox_x0": 97.343964, + "bbox_x1": 105.636917, + "bbox_y0": 5.61257, + "bbox_y1": 20.464701 + } + }, + { + "pk": 227, + "model": "base.region", + "fields": { + "rght": 313, + "code": "TLS", + "name": "Timor-Leste", + "parent": 7, + "level": 3, + "lft": 312, + "tree_id": 90, + "bbox_x0": 124.075439, + "bbox_x1": 127.345337, + "bbox_y0": -9.51337, + "bbox_y1": -8.13741 + } + }, + { + "pk": 228, + "model": "base.region", + "fields": { + "rght": 125, + "code": "TGO", + "name": "Togo", + "parent": 13, + "level": 3, + "lft": 124, + "tree_id": 90, + "bbox_x0": -0.14731, + "bbox_x1": 1.80669, + "bbox_y0": 6.10441, + "bbox_y1": 11.13897 + } + }, + { + "pk": 229, + "model": "base.region", + "fields": { + "rght": 506, + "code": "TKL", + "name": "Tokelau", + "parent": 256, + "level": 2, + "lft": 505, + "tree_id": 90, + "bbox_x0": -172.517136, + "bbox_x1": -171.182083, + "bbox_y0": -9.43378, + "bbox_y1": -8.53288 + } + }, + { + "pk": 230, + "model": "base.region", + "fields": { + "rght": 508, + "code": "TON", + "name": "Tonga", + "parent": 256, + "level": 2, + "lft": 507, + "tree_id": 90, + "bbox_x0": -176.212646, + "bbox_x1": -173.702438, + "bbox_y0": -22.345711, + "bbox_y1": -15.55326 + } + }, + { + "pk": 231, + "model": "base.region", + "fields": { + "rght": 181, + "code": "TTO", + "name": "Trinidad and Tobago", + "parent": 255, + "level": 3, + "lft": 180, + "tree_id": 90, + "bbox_x0": -61.927391, + "bbox_x1": -60.49144, + "bbox_y0": 10.03648, + "bbox_y1": 11.36118 + } + }, + { + "pk": 232, + "model": "base.region", + "fields": { + "rght": 61, + "code": "TUN", + "name": "Tunisia", + "parent": 11, + "level": 3, + "lft": 60, + "tree_id": 90, + "bbox_x0": 7.52481, + "bbox_x1": 11.59827, + "bbox_y0": 30.240431, + "bbox_y1": 37.56712 + } + }, + { + "pk": 233, + "model": "base.region", + "fields": { + "rght": 428, + "code": "TUR", + "name": "Turkey", + "parent": 5, + "level": 2, + "lft": 427, + "tree_id": 90, + "bbox_x0": 25.664101, + "bbox_x1": 44.8297, + "bbox_y0": 35.809971, + "bbox_y1": 42.105499 + } + }, + { + "pk": 234, + "model": "base.region", + "fields": { + "rght": 257, + "code": "TKM", + "name": "Turkmenistan", + "parent": 8, + "level": 3, + "lft": 256, + "tree_id": 90, + "bbox_x0": 52.441429, + "bbox_x1": 66.684303, + "bbox_y0": 35.14109, + "bbox_y1": 42.795551 + } + }, + { + "pk": 235, + "model": "base.region", + "fields": { + "rght": 183, + "code": "TCA", + "name": "Turks and Caicos Islands", + "parent": 255, + "level": 3, + "lft": 182, + "tree_id": 90, + "bbox_x0": -72.483879, + "bbox_x1": -71.08033, + "bbox_y0": 21.170031, + "bbox_y1": 21.97361 + } + }, + { + "pk": 236, + "model": "base.region", + "fields": { + "rght": 510, + "code": "TUV", + "name": "Tuvalu", + "parent": 256, + "level": 2, + "lft": 509, + "tree_id": 90, + "bbox_x0": 176.06488, + "bbox_x1": 179.883789, + "bbox_y0": -10.75045, + "bbox_y1": -5.64198 + } + }, + { + "pk": 237, + "model": "base.region", + "fields": { + "rght": 45, + "code": "UGA", + "name": "Uganda", + "parent": 12, + "level": 3, + "lft": 44, + "tree_id": 90, + "bbox_x0": 29.573549, + "bbox_x1": 35.001251, + "bbox_y0": -1.47849, + "bbox_y1": 4.23403 + } + }, + { + "pk": 238, + "model": "base.region", + "fields": { + "rght": 430, + "code": "UKR", + "name": "Ukraine", + "parent": 5, + "level": 2, + "lft": 429, + "tree_id": 90, + "bbox_x0": 22.128811, + "bbox_x1": 40.218079, + "bbox_y0": 44.390411, + "bbox_y1": 52.375359 + } + }, + { + "pk": 239, + "model": "base.region", + "fields": { + "rght": 460, + "code": "ARE", + "name": "United Arab Emirates", + "parent": 15, + "level": 2, + "lft": 459, + "tree_id": 90, + "bbox_x0": 51.497978, + "bbox_x1": 56.38343, + "bbox_y0": 22.644409, + "bbox_y1": 26.28219 + } + }, + { + "pk": 240, + "model": "base.region", + "fields": { + "rght": 432, + "code": "GBR", + "name": "United Kingdom", + "parent": 5, + "level": 2, + "lft": 431, + "tree_id": 90, + "bbox_x0": -13.41393, + "bbox_x1": 1.76896, + "bbox_y0": 49.16209, + "bbox_y1": 60.854691 + } + }, + { + "pk": 241, + "model": "base.region", + "fields": { + "rght": 47, + "code": "TZA", + "name": "United Republic of Tanzania", + "parent": 12, + "level": 3, + "lft": 46, + "tree_id": 90, + "bbox_x0": 29.32716, + "bbox_x1": 40.443218, + "bbox_y0": -11.74569, + "bbox_y1": -0.99073 + } + }, + { + "pk": 242, + "model": "base.region", + "fields": { + "rght": 185, + "code": "VIR", + "name": "United States Virgin Islands", + "parent": 255, + "level": 3, + "lft": 184, + "tree_id": 90, + "bbox_x0": -65.086281, + "bbox_x1": -64.56517, + "bbox_y0": 17.681721, + "bbox_y1": 18.458139 + } + }, + { + "pk": 243, + "model": "base.region", + "fields": { + "rght": 211, + "code": "USA", + "name": "United States of America", + "parent": 2, + "level": 3, + "lft": 210, + "tree_id": 90, + "bbox_x0": -179.150558, + "bbox_x1": -66.940643, + "bbox_y0": 18.91172, + "bbox_y1": 71.441048 + } + }, + { + "pk": 244, + "model": "base.region", + "fields": { + "rght": 239, + "code": "URY", + "name": "Uruguay", + "parent": 4, + "level": 3, + "lft": 238, + "tree_id": 90, + "bbox_x0": -58.442719, + "bbox_x1": -53.073929, + "bbox_y0": -35.047939, + "bbox_y1": -30.08222 + } + }, + { + "pk": 245, + "model": "base.region", + "fields": { + "rght": 259, + "code": "UZB", + "name": "Uzbekistan", + "parent": 8, + "level": 3, + "lft": 258, + "tree_id": 90, + "bbox_x0": 55.996632, + "bbox_x1": 73.132271, + "bbox_y0": 37.18433, + "bbox_y1": 45.60519 + } + }, + { + "pk": 246, + "model": "base.region", + "fields": { + "rght": 512, + "code": "VUT", + "name": "Vanuatu", + "parent": 256, + "level": 2, + "lft": 511, + "tree_id": 90, + "bbox_x0": 166.524994, + "bbox_x1": 170.234802, + "bbox_y0": -20.25045, + "bbox_y1": -13.07345 + } + }, + { + "pk": 247, + "model": "base.region", + "fields": { + "rght": 241, + "code": "VEN", + "name": "Venezuela (Bolivarian Republic of)", + "parent": 4, + "level": 3, + "lft": 240, + "tree_id": 90, + "bbox_x0": -73.374313, + "bbox_x1": -59.803768, + "bbox_y0": 0.74368, + "bbox_y1": 12.2019 + } + }, + { + "pk": 248, + "model": "base.region", + "fields": { + "rght": 315, + "code": "VNM", + "name": "Viet Nam", + "parent": 7, + "level": 3, + "lft": 314, + "tree_id": 90, + "bbox_x0": 102.144592, + "bbox_x1": 116.521233, + "bbox_y0": 7.3116, + "bbox_y1": 23.39274 + } + }, + { + "pk": 249, + "model": "base.region", + "fields": { + "rght": 514, + "code": "WLF", + "name": "Wallis and Futuna Islands", + "parent": 256, + "level": 2, + "lft": 513, + "tree_id": 90, + "bbox_x0": -178.206787, + "bbox_x1": -176.08287, + "bbox_y0": -14.38779, + "bbox_y1": -13.17343 + } + }, + { + "pk": 250, + "model": "base.region", + "fields": { + "rght": 63, + "code": "ESH", + "name": "Western Sahara", + "parent": 11, + "level": 3, + "lft": 62, + "tree_id": 90, + "bbox_x0": -17.10317, + "bbox_x1": -8.66942, + "bbox_y0": 20.774151, + "bbox_y1": 28.219179 + } + }, + { + "pk": 251, + "model": "base.region", + "fields": { + "rght": 462, + "code": "YEM", + "name": "Yemen", + "parent": 15, + "level": 2, + "lft": 461, + "tree_id": 90, + "bbox_x0": 41.809608, + "bbox_x1": 54.535992, + "bbox_y0": 12.10717, + "bbox_y1": 19.00276 + } + }, + { + "pk": 252, + "model": "base.region", + "fields": { + "rght": 79, + "code": "ZMB", + "name": "Zambia", + "parent": 14, + "level": 3, + "lft": 78, + "tree_id": 90, + "bbox_x0": 21.99938, + "bbox_x1": 33.705711, + "bbox_y0": -18.07947, + "bbox_y1": -8.22436 + } + }, + { + "pk": 253, + "model": "base.region", + "fields": { + "rght": 81, + "code": "ZWE", + "name": "Zimbabwe", + "parent": 14, + "level": 3, + "lft": 80, + "tree_id": 90, + "bbox_x0": 25.23702, + "bbox_x1": 33.056301, + "bbox_y0": -22.41773, + "bbox_y1": -15.60883 + } + }, + { + "pk": 254, + "model": "base.region", + "fields": { + "rght": 243, + "code": "AME", + "name": "Americas", + "parent": null, + "level": 1, + "lft": 128, + "tree_id": 90, + "bbox_x0": -84.114449, + "bbox_x1": -84.097572, + "bbox_y0": 9.9313, + "bbox_y1": 9.94792 + } + }, + { + "pk": 255, + "model": "base.region", + "fields": { + "rght": 186, + "code": "CRB", + "name": "Caribbean", + "parent": 254, + "level": 2, + "lft": 129, + "tree_id": 90, + "bbox_x0": -85.260178, + "bbox_x1": -59.286541, + "bbox_y0": 10.18548, + "bbox_y1": 27.454559 + } + }, + { + "pk": 256, + "model": "base.region", + "fields": { + "rght": 515, + "code": "PAC", + "name": "Pacific", + "parent": null, + "level": 1, + "lft": 464, + "tree_id": 90, + "bbox_x0": 139.572356, + "bbox_x1": -74.531208, + "bbox_y0": -56.267241, + "bbox_y1": 62.0215969 + } + }, + { + "pk": 257, + "model": "base.region", + "fields": { + "rght": 12, + "code": "CFR", + "name": "Central Africa", + "parent": 10, + "level": 2, + "lft": 3, + "tree_id": 90, + "bbox_x0": -25.35874, + "bbox_x1": 63.525379, + "bbox_y0": -46.900452, + "bbox_y1": 37.56712 + } + }, + { + "pk": 258, + "model": "base.region", + "fields": { + "rght": 276, + "code": "EAS", + "name": "East Asia", + "parent": 6, + "level": 2, + "lft": 261, + "tree_id": 90, + "bbox_x0": 19.6381, + "bbox_x1": 180, + "bbox_y0": -12.56111, + "bbox_y1": 82.50045 + } + }, + { + "pk": 259, + "model": "base.region", + "fields": { + "rght": 59, + "code": "SSD", + "name": "South Sudan", + "parent": 11, + "level": 3, + "lft": 58, + "tree_id": 90, + "bbox_x0": 24.15192, + "bbox_x1": 35.947689, + "bbox_y0": 3.48639, + "bbox_y1": 12.21558 + } + }, + { + "model": "base.menuplaceholder", + "pk": 1, + "fields": { + "name": "TOPBAR_MENU" + } + }, + { + "model": "base.menuplaceholder", + "pk": 2, + "fields": { + "name": "TOPBAR_MENU_LEFT" + } + }, + { + "model": "base.menuplaceholder", + "pk": 3, + "fields": { + "name": "TOPBAR_MENU_RIGHT" + } + }, + { + "model": "base.menuplaceholder", + "pk": 4, + "fields": { + "name": "CARDS_MENU" + } + } +] diff --git a/geonode/base/fixtures/profiles_test_data.json b/geonode/base/fixtures/profiles_test_data.json new file mode 100644 index 0000000..8fba186 --- /dev/null +++ b/geonode/base/fixtures/profiles_test_data.json @@ -0,0 +1,56 @@ +[ + { + "fields": { + "date_joined": "2011-06-09 15:15:27", + "email": "ad@m.in", + "first_name": "admin", + "groups": [], + "is_active": true, + "is_staff": true, + "is_superuser": true, + "last_login": "2011-06-09 15:45:34", + "last_name": "", + "password": "pbkdf2_sha256$30000$rjuGt0Obn8on$cxF75frIOSaitNklLZ0IJ/VonUW0fwEFVF96o0M+lGc=", + "user_permissions": [], + "username": "admin" + }, + "model": "people.profile", + "pk": 1 + }, + { + "fields": { + "date_joined": "2011-06-09 15:46:03", + "email": "", + "first_name": "Norman", + "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": "2011-06-09 15:46:03", + "last_name": "Normandy", + "password": "sha1$b0022$1592757da52adeed741051a203fe2d65b6798e83", + "user_permissions": [], + "username": "norman" + }, + "model": "people.profile", + "pk": 2 + }, + { + "fields": { + "date_joined": "2011-06-09 15:46:03", + "email": "", + "first_name": "Test", + "groups": [], + "is_active": true, + "is_staff": false, + "is_superuser": false, + "last_login": "2011-06-09 15:46:03", + "last_name": "User", + "password": "sha1$b0022$1592757da52adeed741051a203fe2d65b6798e83", + "user_permissions": [], + "username": "test_user" + }, + "model": "people.profile", + "pk": 3 + } +] diff --git a/geonode/base/fixtures/regions.json b/geonode/base/fixtures/regions.json new file mode 100644 index 0000000..aac70db --- /dev/null +++ b/geonode/base/fixtures/regions.json @@ -0,0 +1,4405 @@ +[ + { + "pk": 1, + "model": "base.region", + "fields": { + "rght": 516, + "code": "GLO", + "name": "Global", + "parent": null, + "level": 0, + "lft": 1, + "tree_id": 90, + "bbox_x0": -180, + "bbox_x1": 180, + "bbox_y0": -90, + "bbox_y1": 90 + } + }, + { + "pk": 2, + "model": "base.region", + "fields": { + "rght": 212, + "code": "NAM", + "name": "North America", + "parent": 254, + "level": 2, + "lft": 203, + "tree_id": 90, + "bbox_x0": -167.276413, + "bbox_x1": -52.23304, + "bbox_y0": 5.49955, + "bbox_y1": 83.162102 + } + }, + { + "pk": 3, + "model": "base.region", + "fields": { + "rght": 202, + "code": "CAM", + "name": "Central America", + "parent": 254, + "level": 2, + "lft": 187, + "tree_id": 90, + "bbox_x0": -118.867172, + "bbox_x1": -66.869827, + "bbox_y0": -4.23048, + "bbox_y1": 32.71862 + } + }, + { + "pk": 4, + "model": "base.region", + "fields": { + "rght": 242, + "code": "SAM", + "name": "South America", + "parent": 254, + "level": 2, + "lft": 213, + "tree_id": 90, + "bbox_x0": -109.47493, + "bbox_x1": -26.33247, + "bbox_y0": -59.450451, + "bbox_y1": 13.39029 + } + }, + { + "pk": 5, + "model": "base.region", + "fields": { + "rght": 433, + "code": "EUR", + "name": "Europe", + "parent": null, + "level": 1, + "lft": 318, + "tree_id": 90, + "bbox_x0": -31.266001, + "bbox_x1": 39.869301, + "bbox_y0": 27.636311, + "bbox_y1": 81.008797 + } + }, + { + "pk": 6, + "model": "base.region", + "fields": { + "rght": 317, + "code": "ASI", + "name": "Asia", + "parent": null, + "level": 1, + "lft": 246, + "tree_id": 90, + "bbox_x0": 19.6381, + "bbox_x1": 180, + "bbox_y0": -12.56111, + "bbox_y1": 82.50045 + } + }, + { + "pk": 7, + "model": "base.region", + "fields": { + "rght": 316, + "code": "SEA", + "name": "Southeast Asia", + "parent": 6, + "level": 2, + "lft": 293, + "tree_id": 90, + "bbox_x0": 68.0327, + "bbox_x1": 141.021805, + "bbox_y0": -12.56111, + "bbox_y1": 35.504211 + } + }, + { + "pk": 8, + "model": "base.region", + "fields": { + "rght": 260, + "code": "CTA", + "name": "Central Asia", + "parent": 6, + "level": 2, + "lft": 247, + "tree_id": 90, + "bbox_x0": 44.236641, + "bbox_x1": 90.076767, + "bbox_y0": 33.890511, + "bbox_y1": 54.845139 + } + }, + { + "pk": 9, + "model": "base.region", + "fields": { + "rght": 292, + "code": "SAS", + "name": "South Asia", + "parent": 6, + "level": 2, + "lft": 277, + "tree_id": 90, + "bbox_x0": 19.6381, + "bbox_x1": 180, + "bbox_y0": -12.56111, + "bbox_y1": 82.50045 + } + }, + { + "pk": 10, + "model": "base.region", + "fields": { + "rght": 127, + "code": "AFR", + "name": "Africa", + "parent": null, + "level": 1, + "lft": 2, + "tree_id": 90, + "bbox_x0": -25.35874, + "bbox_x1": 63.525379, + "bbox_y0": -46.900452, + "bbox_y1": 37.56712 + } + }, + { + "pk": 11, + "model": "base.region", + "fields": { + "rght": 64, + "code": "NAF", + "name": "North Africa", + "parent": 10, + "level": 2, + "lft": 49, + "tree_id": 90, + "bbox_x0": -17.10317, + "bbox_x1": 38.833801, + "bbox_y0": 3.48639, + "bbox_y1": 37.56712 + } + }, + { + "pk": 12, + "model": "base.region", + "fields": { + "rght": 48, + "code": "EAF", + "name": "East Africa", + "parent": 10, + "level": 2, + "lft": 13, + "tree_id": 90, + "bbox_x0": 22.855089, + "bbox_x1": 63.94656, + "bbox_y0": -25.84763, + "bbox_y1": 17.467039 + } + }, + { + "pk": 13, + "model": "base.region", + "fields": { + "rght": 126, + "code": "WAF", + "name": "West Africa", + "parent": 10, + "level": 2, + "lft": 83, + "tree_id": 90, + "bbox_x0": -26.758421, + "bbox_x1": 24.002661, + "bbox_y0": -9.29925, + "bbox_y1": 27.702801 + } + }, + { + "pk": 14, + "model": "base.region", + "fields": { + "rght": 82, + "code": "SAF", + "name": "Southern Africa", + "parent": 10, + "level": 2, + "lft": 65, + "tree_id": 90, + "bbox_x0": 8.93107, + "bbox_x1": 42.74847, + "bbox_y0": -35.507481, + "bbox_y1": -13.27553 + } + }, + { + "pk": 15, + "model": "base.region", + "fields": { + "rght": 463, + "code": "MES", + "name": "Middle East", + "parent": null, + "level": 1, + "lft": 434, + "tree_id": 90, + "bbox_x0": 24.698099, + "bbox_x1": 63.317459, + "bbox_y0": 12.111, + "bbox_y1": 42.10751 + } + }, + { + "pk": 16, + "model": "base.region", + "fields": { + "rght": 245, + "code": "ANT", + "name": "Antarctica", + "parent": null, + "level": 1, + "lft": 244, + "tree_id": 90, + "bbox_x0": -180, + "bbox_x1": 180, + "bbox_y0": -90, + "bbox_y1": -73 + } + }, + { + "pk": 17, + "model": "base.region", + "fields": { + "rght": 249, + "code": "AFG", + "name": "Afghanistan", + "parent": 8, + "level": 3, + "lft": 248, + "tree_id": 90, + "bbox_x0": 60.478439, + "bbox_x1": 74.879433, + "bbox_y0": 29.37747, + "bbox_y1": 38.483421 + } + }, + { + "pk": 18, + "model": "base.region", + "fields": { + "rght": 320, + "code": "ALA", + "name": "Aland Islands", + "parent": 5, + "level": 2, + "lft": 319, + "tree_id": 90, + "bbox_x0": 19.262711, + "bbox_x1": 21.324409, + "bbox_y0": 59.736301, + "bbox_y1": 60.665581 + } + }, + { + "pk": 19, + "model": "base.region", + "fields": { + "rght": 322, + "code": "ALB", + "name": "Albania", + "parent": 5, + "level": 2, + "lft": 321, + "tree_id": 90, + "bbox_x0": 19.28219, + "bbox_x1": 21.057819, + "bbox_y0": 39.644489, + "bbox_y1": 42.660801 + } + }, + { + "pk": 20, + "model": "base.region", + "fields": { + "rght": 51, + "code": "DZA", + "name": "Algeria", + "parent": 11, + "level": 3, + "lft": 50, + "tree_id": 90, + "bbox_x0": -8.67386, + "bbox_x1": 11.97955, + "bbox_y0": 18.96002, + "bbox_y1": 37.093731 + } + }, + { + "pk": 21, + "model": "base.region", + "fields": { + "rght": 466, + "code": "ASM", + "name": "American Samoa", + "parent": 256, + "level": 2, + "lft": 465, + "tree_id": 90, + "bbox_x0": -171.091873, + "bbox_x1": -169.416077, + "bbox_y0": -14.38247, + "bbox_y1": -11.04969 + } + }, + { + "pk": 22, + "model": "base.region", + "fields": { + "rght": 324, + "code": "AND", + "name": "Andorra", + "parent": 5, + "level": 2, + "lft": 323, + "tree_id": 90, + "bbox_x0": 1.41382, + "bbox_x1": 1.78659, + "bbox_y0": 42.42873, + "bbox_y1": 42.65601 + } + }, + { + "pk": 23, + "model": "base.region", + "fields": { + "rght": 85, + "code": "AGO", + "name": "Angola", + "parent": 13, + "level": 3, + "lft": 84, + "tree_id": 90, + "bbox_x0": 11.6792, + "bbox_x1": 24.082109, + "bbox_y0": -18.04207, + "bbox_y1": -4.37259 + } + }, + { + "pk": 24, + "model": "base.region", + "fields": { + "rght": 131, + "code": "AIA", + "name": "Anguilla", + "parent": 255, + "level": 3, + "lft": 130, + "tree_id": 90, + "bbox_x0": -63.434872, + "bbox_x1": -62.916199, + "bbox_y0": 18.149549, + "bbox_y1": 18.61278 + } + }, + { + "pk": 25, + "model": "base.region", + "fields": { + "rght": 133, + "code": "ATG", + "name": "Antigua and Barbuda", + "parent": 255, + "level": 3, + "lft": 132, + "tree_id": 90, + "bbox_x0": -62.352402, + "bbox_x1": -61.659081, + "bbox_y0": 16.927219, + "bbox_y1": 17.72938 + } + }, + { + "pk": 26, + "model": "base.region", + "fields": { + "rght": 215, + "code": "ARG", + "name": "Argentina", + "parent": 4, + "level": 3, + "lft": 214, + "tree_id": 90, + "bbox_x0": -73.577782, + "bbox_x1": -53.637539, + "bbox_y0": -55.057362, + "bbox_y1": -21.78126 + } + }, + { + "pk": 27, + "model": "base.region", + "fields": { + "rght": 326, + "code": "ARM", + "name": "Armenia", + "parent": 5, + "level": 2, + "lft": 325, + "tree_id": 90, + "bbox_x0": 43.449749, + "bbox_x1": 46.630039, + "bbox_y0": 38.830521, + "bbox_y1": 41.30183 + } + }, + { + "pk": 28, + "model": "base.region", + "fields": { + "rght": 135, + "code": "ABW", + "name": "Aruba", + "parent": 255, + "level": 3, + "lft": 134, + "tree_id": 90, + "bbox_x0": -70.0611, + "bbox_x1": -69.8669, + "bbox_y0": 12.4061, + "bbox_y1": 12.6306 + } + }, + { + "pk": 29, + "model": "base.region", + "fields": { + "rght": 468, + "code": "AUS", + "name": "Australia", + "parent": 256, + "level": 2, + "lft": 467, + "tree_id": 90, + "bbox_x0": 112.921112, + "bbox_x1": 159.278717, + "bbox_y0": -54.640301, + "bbox_y1": -9.22882 + } + }, + { + "pk": 30, + "model": "base.region", + "fields": { + "rght": 328, + "code": "AUT", + "name": "Austria", + "parent": 5, + "level": 2, + "lft": 327, + "tree_id": 90, + "bbox_x0": 9.53079, + "bbox_x1": 17.160749, + "bbox_y0": 46.372299, + "bbox_y1": 49.02071 + } + }, + { + "pk": 31, + "model": "base.region", + "fields": { + "rght": 330, + "code": "AZE", + "name": "Azerbaijan", + "parent": 5, + "level": 2, + "lft": 329, + "tree_id": 90, + "bbox_x0": 44.7719, + "bbox_x1": 50.6078, + "bbox_y0": 38.3970, + "bbox_y1": 41.9056 + } + }, + { + "pk": 32, + "model": "base.region", + "fields": { + "rght": 137, + "code": "BHS", + "name": "Bahamas", + "parent": 255, + "level": 3, + "lft": 136, + "tree_id": 90, + "bbox_x0": -80.499229, + "bbox_x1": -72.649513, + "bbox_y0": 20.916059, + "bbox_y1": 27.933781 + } + }, + { + "pk": 33, + "model": "base.region", + "fields": { + "rght": 436, + "code": "BHR", + "name": "Bahrain", + "parent": 15, + "level": 2, + "lft": 435, + "tree_id": 90, + "bbox_x0": 50.385799, + "bbox_x1": 50.828499, + "bbox_y0": 25.5422, + "bbox_y1": 26.292391 + } + }, + { + "pk": 34, + "model": "base.region", + "fields": { + "rght": 279, + "code": "BGD", + "name": "Bangladesh", + "parent": 9, + "level": 3, + "lft": 278, + "tree_id": 90, + "bbox_x0": 88.028198, + "bbox_x1": 92.673599, + "bbox_y0": 20.585199, + "bbox_y1": 26.631701 + } + }, + { + "pk": 35, + "model": "base.region", + "fields": { + "rght": 139, + "code": "BRB", + "name": "Barbados", + "parent": 255, + "level": 3, + "lft": 138, + "tree_id": 90, + "bbox_x0": -59.648918, + "bbox_x1": -59.420368, + "bbox_y0": 13.03984, + "bbox_y1": 13.32725 + } + }, + { + "pk": 36, + "model": "base.region", + "fields": { + "rght": 332, + "code": "BLR", + "name": "Belarus", + "parent": 5, + "level": 2, + "lft": 331, + "tree_id": 90, + "bbox_x0": 23.17679, + "bbox_x1": 32.77071, + "bbox_y0": 51.256401, + "bbox_y1": 56.16571 + } + }, + { + "pk": 37, + "model": "base.region", + "fields": { + "rght": 334, + "code": "BEL", + "name": "Belgium", + "parent": 5, + "level": 2, + "lft": 333, + "tree_id": 90, + "bbox_x0": 2.54563, + "bbox_x1": 6.40791, + "bbox_y0": 49.496899, + "bbox_y1": 51.505081 + } + }, + { + "pk": 38, + "model": "base.region", + "fields": { + "rght": 189, + "code": "BLZ", + "name": "Belize", + "parent": 3, + "level": 3, + "lft": 188, + "tree_id": 90, + "bbox_x0": -89.224823, + "bbox_x1": -87.468132, + "bbox_y0": 15.8893, + "bbox_y1": 18.49655 + } + }, + { + "pk": 39, + "model": "base.region", + "fields": { + "rght": 87, + "code": "BEN", + "name": "Benin", + "parent": 13, + "level": 3, + "lft": 86, + "tree_id": 90, + "bbox_x0": 0.77456, + "bbox_x1": 3.8517, + "bbox_y0": 6.22574, + "bbox_y1": 12.41834 + } + }, + { + "pk": 40, + "model": "base.region", + "fields": { + "rght": 141, + "code": "BMU", + "name": "Bermuda", + "parent": 255, + "level": 3, + "lft": 140, + "tree_id": 90, + "bbox_x0": -64.896042, + "bbox_x1": -64.642952, + "bbox_y0": 32.230709, + "bbox_y1": 32.393829 + } + }, + { + "pk": 41, + "model": "base.region", + "fields": { + "rght": 281, + "code": "BTN", + "name": "Bhutan", + "parent": 9, + "level": 3, + "lft": 280, + "tree_id": 90, + "bbox_x0": 88.759521, + "bbox_x1": 92.125023, + "bbox_y0": 26.7075, + "bbox_y1": 28.3235 + } + }, + { + "pk": 42, + "model": "base.region", + "fields": { + "rght": 217, + "code": "BOL", + "name": "Bolivia", + "parent": 4, + "level": 3, + "lft": 216, + "tree_id": 90, + "bbox_x0": -69.640762, + "bbox_x1": -57.458092, + "bbox_y0": -22.89613, + "bbox_y1": -9.68056 + } + }, + { + "pk": 43, + "model": "base.region", + "fields": { + "rght": 336, + "code": "BIH", + "name": "Bosnia and Herzegovina", + "parent": 5, + "level": 2, + "lft": 335, + "tree_id": 90, + "bbox_x0": 15.74909, + "bbox_x1": 19.62907, + "bbox_y0": 42.56451, + "bbox_y1": 45.276001 + } + }, + { + "pk": 44, + "model": "base.region", + "fields": { + "rght": 67, + "code": "BWA", + "name": "Botswana", + "parent": 14, + "level": 3, + "lft": 66, + "tree_id": 90, + "bbox_x0": 19.999531, + "bbox_x1": 29.360781, + "bbox_y0": -26.90724, + "bbox_y1": -17.780809 + } + }, + { + "pk": 45, + "model": "base.region", + "fields": { + "rght": 219, + "code": "BRA", + "name": "Brazil", + "parent": 4, + "level": 3, + "lft": 218, + "tree_id": 90, + "bbox_x0": -73.985527, + "bbox_x1": -28.839041, + "bbox_y0": -33.750702, + "bbox_y1": 5.26486 + } + }, + { + "pk": 46, + "model": "base.region", + "fields": { + "rght": 143, + "code": "VGB", + "name": "British Virgin Islands", + "parent": 255, + "level": 3, + "lft": 142, + "tree_id": 90, + "bbox_x0": -64.783012, + "bbox_x1": -64.268761, + "bbox_y0": 18.312731, + "bbox_y1": 18.757219 + } + }, + { + "pk": 47, + "model": "base.region", + "fields": { + "rght": 295, + "code": "BRN", + "name": "Brunei Darussalam", + "parent": 7, + "level": 3, + "lft": 294, + "tree_id": 90, + "bbox_x0": 114.071457, + "bbox_x1": 115.359451, + "bbox_y0": 4.00309, + "bbox_y1": 5.04717 + } + }, + { + "pk": 48, + "model": "base.region", + "fields": { + "rght": 338, + "code": "BGR", + "name": "Bulgaria", + "parent": 5, + "level": 2, + "lft": 337, + "tree_id": 90, + "bbox_x0": 22.35741, + "bbox_x1": 28.60882, + "bbox_y0": 41.235931, + "bbox_y1": 44.227261 + } + }, + { + "pk": 49, + "model": "base.region", + "fields": { + "rght": 89, + "code": "BFA", + "name": "Burkina Faso", + "parent": 13, + "level": 3, + "lft": 88, + "tree_id": 90, + "bbox_x0": -5.51891, + "bbox_x1": 2.40539, + "bbox_y0": 9.4011, + "bbox_y1": 15.08259 + } + }, + { + "pk": 50, + "model": "base.region", + "fields": { + "rght": 15, + "code": "BDI", + "name": "Burundi", + "parent": 12, + "level": 3, + "lft": 14, + "tree_id": 90, + "bbox_x0": 28.993071, + "bbox_x1": 30.847719, + "bbox_y0": -4.46571, + "bbox_y1": -2.31012 + } + }, + { + "pk": 51, + "model": "base.region", + "fields": { + "rght": 297, + "code": "KHM", + "name": "Cambodia", + "parent": 7, + "level": 3, + "lft": 296, + "tree_id": 90, + "bbox_x0": 102.340012, + "bbox_x1": 107.627724, + "bbox_y0": 9.28325, + "bbox_y1": 14.6864 + } + }, + { + "pk": 52, + "model": "base.region", + "fields": { + "rght": 91, + "code": "CMR", + "name": "Cameroon", + "parent": 13, + "level": 3, + "lft": 90, + "tree_id": 90, + "bbox_x0": 8.49477, + "bbox_x1": 16.19211, + "bbox_y0": 1.65254, + "bbox_y1": 13.07805 + } + }, + { + "pk": 53, + "model": "base.region", + "fields": { + "rght": 205, + "code": "CAN", + "name": "Canada", + "parent": 2, + "level": 3, + "lft": 204, + "tree_id": 90, + "bbox_x0": -141.002701, + "bbox_x1": -52.620201, + "bbox_y0": 41.681019, + "bbox_y1": 83.110619 + } + }, + { + "pk": 54, + "model": "base.region", + "fields": { + "rght": 93, + "code": "CPV", + "name": "Cape Verde", + "parent": 13, + "level": 3, + "lft": 92, + "tree_id": 90, + "bbox_x0": -25.35874, + "bbox_x1": -22.666201, + "bbox_y0": 14.80221, + "bbox_y1": 17.19717 + } + }, + { + "pk": 55, + "model": "base.region", + "fields": { + "rght": 145, + "code": "CYM", + "name": "Cayman Islands", + "parent": 255, + "level": 3, + "lft": 144, + "tree_id": 90, + "bbox_x0": -81.420593, + "bbox_x1": -79.722321, + "bbox_y0": 19.262659, + "bbox_y1": 19.75738 + } + }, + { + "pk": 56, + "model": "base.region", + "fields": { + "rght": 5, + "code": "CAF", + "name": "Central African Republic", + "parent": 257, + "level": 3, + "lft": 4, + "tree_id": 90, + "bbox_x0": 14.42009, + "bbox_x1": 27.463421, + "bbox_y0": 2.22051, + "bbox_y1": 11.00756 + } + }, + { + "pk": 57, + "model": "base.region", + "fields": { + "rght": 7, + "code": "TCD", + "name": "Chad", + "parent": 257, + "level": 3, + "lft": 6, + "tree_id": 90, + "bbox_x0": 13.47592, + "bbox_x1": 24.00161, + "bbox_y0": 7.44237, + "bbox_y1": 23.478239 + } + }, + { + "pk": 58, + "model": "base.region", + "fields": { + "rght": 340, + "code": "CIL", + "name": "Channel Islands", + "parent": 5, + "level": 2, + "lft": 339, + "tree_id": 90, + "bbox_x0": -2.67545, + "bbox_x1": -2.01129, + "bbox_y0": 49.16209, + "bbox_y1": 49.7393 + } + }, + { + "pk": 59, + "model": "base.region", + "fields": { + "rght": 221, + "code": "CHL", + "name": "Chile", + "parent": 4, + "level": 3, + "lft": 220, + "tree_id": 90, + "bbox_x0": -109.47493, + "bbox_x1": -66.417549, + "bbox_y0": -56.533779, + "bbox_y1": -17.507549 + } + }, + { + "pk": 60, + "model": "base.region", + "fields": { + "rght": 263, + "code": "CHN", + "name": "China", + "parent": 258, + "level": 3, + "lft": 262, + "tree_id": 90, + "bbox_x0": 73.557701, + "bbox_x1": 134.773605, + "bbox_y0": 15.77539, + "bbox_y1": 53.5606 + } + }, + { + "pk": 61, + "model": "base.region", + "fields": { + "rght": 265, + "code": "HKG", + "name": "China - Hong Kong", + "parent": 258, + "level": 3, + "lft": 264, + "tree_id": 90, + "bbox_x0": 113.835083, + "bbox_x1": 114.441788, + "bbox_y0": 22.153549, + "bbox_y1": 22.56204 + } + }, + { + "pk": 62, + "model": "base.region", + "fields": { + "rght": 267, + "code": "MAC", + "name": "China - Macao", + "parent": 258, + "level": 3, + "lft": 266, + "tree_id": 90, + "bbox_x0": 113.528351, + "bbox_x1": 113.598297, + "bbox_y0": 22.10977, + "bbox_y1": 22.21697 + } + }, + { + "pk": 63, + "model": "base.region", + "fields": { + "rght": 223, + "code": "COL", + "name": "Colombia", + "parent": 4, + "level": 3, + "lft": 222, + "tree_id": 90, + "bbox_x0": -81.728111, + "bbox_x1": -66.869827, + "bbox_y0": -4.23048, + "bbox_y1": 13.39029 + } + }, + { + "pk": 64, + "model": "base.region", + "fields": { + "rght": 17, + "code": "COM", + "name": "Comoros", + "parent": 12, + "level": 3, + "lft": 16, + "tree_id": 90, + "bbox_x0": 43.215778, + "bbox_x1": 44.538219, + "bbox_y0": -12.41382, + "bbox_y1": -11.36238 + } + }, + { + "pk": 65, + "model": "base.region", + "fields": { + "rght": 9, + "code": "COG", + "name": "Congo", + "parent": 257, + "level": 3, + "lft": 8, + "tree_id": 90, + "bbox_x0": 11.205, + "bbox_x1": 18.64983, + "bbox_y0": -5.02831, + "bbox_y1": 3.70308 + } + }, + { + "pk": 66, + "model": "base.region", + "fields": { + "rght": 470, + "code": "COK", + "name": "Cook Islands", + "parent": 256, + "level": 2, + "lft": 469, + "tree_id": 90, + "bbox_x0": -165.858093, + "bbox_x1": -157.312119, + "bbox_y0": -21.94416, + "bbox_y1": -8.94402 + } + }, + { + "pk": 67, + "model": "base.region", + "fields": { + "rght": 191, + "code": "CRI", + "name": "Costa Rica", + "parent": 3, + "level": 3, + "lft": 190, + "tree_id": 90, + "bbox_x0": -87.083778, + "bbox_x1": -82.556, + "bbox_y0": 5.49955, + "bbox_y1": 11.21681 + } + }, + { + "pk": 68, + "model": "base.region", + "fields": { + "rght": 95, + "code": "CIV", + "name": "Cote d'Ivoire", + "parent": 13, + "level": 3, + "lft": 94, + "tree_id": 90, + "bbox_x0": -8.6017249, + "bbox_x1": -2.4930309, + "bbox_y0": 4.1642077, + "bbox_y1": 10.740015 + } + }, + { + "pk": 69, + "model": "base.region", + "fields": { + "rght": 342, + "code": "HRV", + "name": "Croatia", + "parent": 5, + "level": 2, + "lft": 341, + "tree_id": 90, + "bbox_x0": 13.48972, + "bbox_x1": 19.44722, + "bbox_y0": 42.392208, + "bbox_y1": 46.554981 + } + }, + { + "pk": 70, + "model": "base.region", + "fields": { + "rght": 147, + "code": "CUB", + "name": "Cuba", + "parent": 255, + "level": 3, + "lft": 146, + "tree_id": 90, + "bbox_x0": -84.957428, + "bbox_x1": -74.131783, + "bbox_y0": 19.828079, + "bbox_y1": 23.283779 + } + }, + { + "pk": 71, + "model": "base.region", + "fields": { + "rght": 344, + "code": "CYP", + "name": "Cyprus", + "parent": 5, + "level": 2, + "lft": 343, + "tree_id": 90, + "bbox_x0": 32.27309, + "bbox_x1": 34.597919, + "bbox_y0": 34.563511, + "bbox_y1": 35.701542 + } + }, + { + "pk": 72, + "model": "base.region", + "fields": { + "rght": 346, + "code": "CZE", + "name": "Czech Republic", + "parent": 5, + "level": 2, + "lft": 345, + "tree_id": 90, + "bbox_x0": 12.0905901, + "bbox_x1": 18.859216, + "bbox_y0": 48.5518144, + "bbox_y1": 51.0557036 + } + }, + { + "pk": 73, + "model": "base.region", + "fields": { + "rght": 269, + "code": "PRK", + "name": "Democratic People's Republic of Korea", + "parent": 258, + "level": 3, + "lft": 268, + "tree_id": 90, + "bbox_x0": 124.182739, + "bbox_x1": 130.674713, + "bbox_y0": 37.632881, + "bbox_y1": 43.006001 + } + }, + { + "pk": 74, + "model": "base.region", + "fields": { + "rght": 11, + "code": "COD", + "name": "Democratic Republic of the Congo", + "parent": 257, + "level": 3, + "lft": 10, + "tree_id": 90, + "bbox_x0": 12.20663, + "bbox_x1": 31.30591, + "bbox_y0": -13.45567, + "bbox_y1": 5.38609 + } + }, + { + "pk": 75, + "model": "base.region", + "fields": { + "rght": 348, + "code": "DNK", + "name": "Denmark", + "parent": 5, + "level": 2, + "lft": 347, + "tree_id": 90, + "bbox_x0": 8.07472, + "bbox_x1": 15.19324, + "bbox_y0": 54.559132, + "bbox_y1": 57.751949 + } + }, + { + "pk": 76, + "model": "base.region", + "fields": { + "rght": 19, + "code": "DJI", + "name": "Djibouti", + "parent": 12, + "level": 3, + "lft": 18, + "tree_id": 90, + "bbox_x0": 41.773441, + "bbox_x1": 43.450459, + "bbox_y0": 10.90991, + "bbox_y1": 12.70683 + } + }, + { + "pk": 77, + "model": "base.region", + "fields": { + "rght": 149, + "code": "DMA", + "name": "Dominica", + "parent": 255, + "level": 3, + "lft": 148, + "tree_id": 90, + "bbox_x0": -61.4841, + "bbox_x1": -61.244148, + "bbox_y0": 15.20168, + "bbox_y1": 15.6318 + } + }, + { + "pk": 78, + "model": "base.region", + "fields": { + "rght": 151, + "code": "DOM", + "name": "Dominican Republic", + "parent": 255, + "level": 3, + "lft": 150, + "tree_id": 90, + "bbox_x0": -72.003479, + "bbox_x1": -68.319992, + "bbox_y0": 17.469299, + "bbox_y1": 19.92985 + } + }, + { + "pk": 79, + "model": "base.region", + "fields": { + "rght": 225, + "code": "ECU", + "name": "Ecuador", + "parent": 4, + "level": 3, + "lft": 224, + "tree_id": 90, + "bbox_x0": -91.66124, + "bbox_x1": -75.200073, + "bbox_y0": -5.01734, + "bbox_y1": 1.45421 + } + }, + { + "pk": 80, + "model": "base.region", + "fields": { + "rght": 53, + "code": "EGY", + "name": "Egypt", + "parent": 11, + "level": 3, + "lft": 52, + "tree_id": 90, + "bbox_x0": 24.698099, + "bbox_x1": 36.89468, + "bbox_y0": 22, + "bbox_y1": 31.674179 + } + }, + { + "pk": 81, + "model": "base.region", + "fields": { + "rght": 193, + "code": "SLV", + "name": "El Salvador", + "parent": 3, + "level": 3, + "lft": 192, + "tree_id": 90, + "bbox_x0": -90.12867, + "bbox_x1": -87.682869, + "bbox_y0": 13.14867, + "bbox_y1": 14.44506 + } + }, + { + "pk": 82, + "model": "base.region", + "fields": { + "rght": 97, + "code": "GNQ", + "name": "Equatorial Guinea", + "parent": 13, + "level": 3, + "lft": 96, + "tree_id": 90, + "bbox_x0": 5.60236, + "bbox_x1": 11.33572, + "bbox_y0": -1.48378, + "bbox_y1": 3.78597 + } + }, + { + "pk": 83, + "model": "base.region", + "fields": { + "rght": 21, + "code": "ERI", + "name": "Eritrea", + "parent": 12, + "level": 3, + "lft": 20, + "tree_id": 90, + "bbox_x0": 36.43877, + "bbox_x1": 43.14864, + "bbox_y0": 12.35956, + "bbox_y1": 18.00308 + } + }, + { + "pk": 84, + "model": "base.region", + "fields": { + "rght": 350, + "code": "EST", + "name": "Estonia", + "parent": 5, + "level": 2, + "lft": 349, + "tree_id": 90, + "bbox_x0": 21.771851, + "bbox_x1": 28.20989, + "bbox_y0": 57.509312, + "bbox_y1": 59.685749 + } + }, + { + "pk": 85, + "model": "base.region", + "fields": { + "rght": 23, + "code": "ETH", + "name": "Ethiopia", + "parent": 12, + "level": 3, + "lft": 22, + "tree_id": 90, + "bbox_x0": 32.99992, + "bbox_x1": 47.986172, + "bbox_y0": 3.40242, + "bbox_y1": 14.89218 + } + }, + { + "pk": 86, + "model": "base.region", + "fields": { + "rght": 352, + "code": "FRO", + "name": "Faeroe Islands", + "parent": 5, + "level": 2, + "lft": 351, + "tree_id": 90, + "bbox_x0": -7.68124, + "bbox_x1": -6.25861, + "bbox_y0": 61.394932, + "bbox_y1": 62.400742 + } + }, + { + "pk": 87, + "model": "base.region", + "fields": { + "rght": 227, + "code": "FLK", + "name": "Falkland Islands (Malvinas)", + "parent": 4, + "level": 3, + "lft": 226, + "tree_id": 90, + "bbox_x0": -61.43404, + "bbox_x1": -57.712479, + "bbox_y0": -52.900581, + "bbox_y1": -50.966221 + } + }, + { + "pk": 88, + "model": "base.region", + "fields": { + "rght": 472, + "code": "FJI", + "name": "Fiji", + "parent": 256, + "level": 2, + "lft": 471, + "tree_id": 90, + "bbox_x0": 174.866196, + "bbox_x1": -178.203156, + "bbox_y0": -21.01712, + "bbox_y1": -12.46622 + } + }, + { + "pk": 89, + "model": "base.region", + "fields": { + "rght": 354, + "code": "FIN", + "name": "Finland", + "parent": 5, + "level": 2, + "lft": 353, + "tree_id": 90, + "bbox_x0": 20.548571, + "bbox_x1": 31.586201, + "bbox_y0": 59.764881, + "bbox_y1": 70.092308 + } + }, + { + "pk": 90, + "model": "base.region", + "fields": { + "rght": 356, + "code": "FRA", + "name": "France", + "parent": 5, + "level": 2, + "lft": 355, + "tree_id": 90, + "bbox_x0": -5.1406, + "bbox_x1": 9.55932, + "bbox_y0": 41.33374, + "bbox_y1": 51.089062 + } + }, + { + "pk": 91, + "model": "base.region", + "fields": { + "rght": 229, + "code": "GUF", + "name": "French Guiana", + "parent": 4, + "level": 3, + "lft": 228, + "tree_id": 90, + "bbox_x0": -54.542511, + "bbox_x1": -51.613941, + "bbox_y0": 2.12709, + "bbox_y1": 5.77649 + } + }, + { + "pk": 92, + "model": "base.region", + "fields": { + "rght": 474, + "code": "PYF", + "name": "French Polynesia", + "parent": 256, + "level": 2, + "lft": 473, + "tree_id": 90, + "bbox_x0": -154.700485, + "bbox_x1": -108.87291, + "bbox_y0": -27.65357, + "bbox_y1": 10.35983 + } + }, + { + "pk": 93, + "model": "base.region", + "fields": { + "rght": 99, + "code": "GAB", + "name": "Gabon", + "parent": 13, + "level": 3, + "lft": 98, + "tree_id": 90, + "bbox_x0": 8.69547, + "bbox_x1": 14.50234, + "bbox_y0": -3.9788, + "bbox_y1": 2.32261 + } + }, + { + "pk": 94, + "model": "base.region", + "fields": { + "rght": 101, + "code": "GMB", + "name": "Gambia", + "parent": 13, + "level": 3, + "lft": 100, + "tree_id": 90, + "bbox_x0": -16.82506, + "bbox_x1": -13.7978, + "bbox_y0": 13.06425, + "bbox_y1": 13.82657 + } + }, + { + "pk": 95, + "model": "base.region", + "fields": { + "rght": 358, + "code": "GEO", + "name": "Georgia", + "parent": 5, + "level": 2, + "lft": 357, + "tree_id": 90, + "bbox_x0": 40.01022, + "bbox_x1": 46.721359, + "bbox_y0": 41.038502, + "bbox_y1": 43.584549 + } + }, + { + "pk": 96, + "model": "base.region", + "fields": { + "rght": 360, + "code": "DEU", + "name": "Germany", + "parent": 5, + "level": 2, + "lft": 359, + "tree_id": 90, + "bbox_x0": 5.86624, + "bbox_x1": 15.04205, + "bbox_y0": 47.27021, + "bbox_y1": 55.05814 + } + }, + { + "pk": 97, + "model": "base.region", + "fields": { + "rght": 103, + "code": "GHA", + "name": "Ghana", + "parent": 13, + "level": 3, + "lft": 102, + "tree_id": 90, + "bbox_x0": -3.25542, + "bbox_x1": 1.19178, + "bbox_y0": 4.73672, + "bbox_y1": 11.1733 + } + }, + { + "pk": 98, + "model": "base.region", + "fields": { + "rght": 362, + "code": "GIB", + "name": "Gibraltar", + "parent": 5, + "level": 2, + "lft": 361, + "tree_id": 90, + "bbox_x0": -5.3579, + "bbox_x1": -5.33867, + "bbox_y0": 36.108219, + "bbox_y1": 36.15593 + } + }, + { + "pk": 99, + "model": "base.region", + "fields": { + "rght": 364, + "code": "GRC", + "name": "Greece", + "parent": 5, + "level": 2, + "lft": 363, + "tree_id": 90, + "bbox_x0": 19.37431, + "bbox_x1": 29.70056, + "bbox_y0": 34.809502, + "bbox_y1": 41.757111 + } + }, + { + "pk": 100, + "model": "base.region", + "fields": { + "rght": 207, + "code": "GRL", + "name": "Greenland", + "parent": 2, + "level": 3, + "lft": 206, + "tree_id": 90, + "bbox_x0": -73.263474, + "bbox_x1": -11.31232, + "bbox_y0": 59.777271, + "bbox_y1": 83.627419 + } + }, + { + "pk": 101, + "model": "base.region", + "fields": { + "rght": 153, + "code": "GRD", + "name": "Grenada", + "parent": 255, + "level": 3, + "lft": 152, + "tree_id": 90, + "bbox_x0": -61.79998, + "bbox_x1": -61.376362, + "bbox_y0": 11.98288, + "bbox_y1": 12.5415 + } + }, + { + "pk": 102, + "model": "base.region", + "fields": { + "rght": 155, + "code": "GLP", + "name": "Guadeloupe", + "parent": 255, + "level": 3, + "lft": 154, + "tree_id": 90, + "bbox_x0": -61.807159, + "bbox_x1": -61, + "bbox_y0": 15.83097, + "bbox_y1": 16.51684 + } + }, + { + "pk": 103, + "model": "base.region", + "fields": { + "rght": 476, + "code": "GUM", + "name": "Guam", + "parent": 256, + "level": 2, + "lft": 475, + "tree_id": 90, + "bbox_x0": 144.619263, + "bbox_x1": 144.953995, + "bbox_y0": 13.24059, + "bbox_y1": 13.65232 + } + }, + { + "pk": 104, + "model": "base.region", + "fields": { + "rght": 195, + "code": "GTM", + "name": "Guatemala", + "parent": 3, + "level": 3, + "lft": 194, + "tree_id": 90, + "bbox_x0": -92.241432, + "bbox_x1": -88.22319, + "bbox_y0": 13.7373, + "bbox_y1": 17.815201 + } + }, + { + "pk": 105, + "model": "base.region", + "fields": { + "rght": 366, + "code": "GGY", + "name": "Guernsey", + "parent": 5, + "level": 2, + "lft": 365, + "tree_id": 90, + "bbox_x0": -2.67545, + "bbox_x1": -2.16382, + "bbox_y0": 49.405762, + "bbox_y1": 49.7393 + } + }, + { + "pk": 106, + "model": "base.region", + "fields": { + "rght": 105, + "code": "GIN", + "name": "Guinea", + "parent": 13, + "level": 3, + "lft": 104, + "tree_id": 90, + "bbox_x0": -15.08625, + "bbox_x1": -7.64106, + "bbox_y0": 7.19355, + "bbox_y1": 12.67621 + } + }, + { + "pk": 107, + "model": "base.region", + "fields": { + "rght": 107, + "code": "GNB", + "name": "Guinea-Bissau", + "parent": 13, + "level": 3, + "lft": 106, + "tree_id": 90, + "bbox_x0": -16.717529, + "bbox_x1": -13.63652, + "bbox_y0": 10.85997, + "bbox_y1": 12.68078 + } + }, + { + "pk": 108, + "model": "base.region", + "fields": { + "rght": 231, + "code": "GUY", + "name": "Guyana", + "parent": 4, + "level": 3, + "lft": 230, + "tree_id": 90, + "bbox_x0": -61.396271, + "bbox_x1": -56.480251, + "bbox_y0": 1.17508, + "bbox_y1": 8.55756 + } + }, + { + "pk": 109, + "model": "base.region", + "fields": { + "rght": 157, + "code": "HTI", + "name": "Haiti", + "parent": 255, + "level": 3, + "lft": 156, + "tree_id": 90, + "bbox_x0": -74.478592, + "bbox_x1": -71.61335, + "bbox_y0": 18.02103, + "bbox_y1": 20.08782 + } + }, + { + "pk": 110, + "model": "base.region", + "fields": { + "rght": 368, + "code": "VAT", + "name": "Holy See (Vatican City)", + "parent": 5, + "level": 2, + "lft": 367, + "tree_id": 90, + "bbox_x0": 12.44584, + "bbox_x1": 12.45842, + "bbox_y0": 41.900211, + "bbox_y1": 41.907459 + } + }, + { + "pk": 111, + "model": "base.region", + "fields": { + "rght": 197, + "code": "HND", + "name": "Honduras", + "parent": 3, + "level": 3, + "lft": 196, + "tree_id": 90, + "bbox_x0": -89.350792, + "bbox_x1": -82.499527, + "bbox_y0": 12.98241, + "bbox_y1": 17.450451 + } + }, + { + "pk": 112, + "model": "base.region", + "fields": { + "rght": 370, + "code": "HUN", + "name": "Hungary", + "parent": 5, + "level": 2, + "lft": 369, + "tree_id": 90, + "bbox_x0": 16.11335, + "bbox_x1": 22.89657, + "bbox_y0": 45.737061, + "bbox_y1": 48.585258 + } + }, + { + "pk": 113, + "model": "base.region", + "fields": { + "rght": 372, + "code": "ISL", + "name": "Iceland", + "parent": 5, + "level": 2, + "lft": 371, + "tree_id": 90, + "bbox_x0": -24.54652, + "bbox_x1": -13.49416, + "bbox_y0": 63.295952, + "bbox_y1": 66.566193 + } + }, + { + "pk": 114, + "model": "base.region", + "fields": { + "rght": 283, + "code": "IND", + "name": "India", + "parent": 9, + "level": 3, + "lft": 282, + "tree_id": 90, + "bbox_x0": 68.032318, + "bbox_x1": 97.403023, + "bbox_y0": 6.7471, + "bbox_y1": 36.261688 + } + }, + { + "pk": 115, + "model": "base.region", + "fields": { + "rght": 299, + "code": "IDN", + "name": "Indonesia", + "parent": 7, + "level": 3, + "lft": 298, + "tree_id": 90, + "bbox_x0": 94.969833, + "bbox_x1": 141.021805, + "bbox_y0": -11.00485, + "bbox_y1": 6.07573 + } + }, + { + "pk": 116, + "model": "base.region", + "fields": { + "rght": 438, + "code": "IRN", + "name": "Iran", + "parent": 15, + "level": 2, + "lft": 437, + "tree_id": 90, + "bbox_x0": 44.047249, + "bbox_x1": 63.317459, + "bbox_y0": 25.064079, + "bbox_y1": 39.777222 + } + }, + { + "pk": 117, + "model": "base.region", + "fields": { + "rght": 440, + "code": "IRQ", + "name": "Iraq", + "parent": 15, + "level": 2, + "lft": 439, + "tree_id": 90, + "bbox_x0": 38.804001, + "bbox_x1": 48.575699, + "bbox_y0": 29.103001, + "bbox_y1": 37.378052 + } + }, + { + "pk": 118, + "model": "base.region", + "fields": { + "rght": 374, + "code": "IRL", + "name": "Ireland", + "parent": 5, + "level": 2, + "lft": 373, + "tree_id": 90, + "bbox_x0": -10.61834, + "bbox_x1": -5.9975, + "bbox_y0": 51.424511, + "bbox_y1": 55.436211 + } + }, + { + "pk": 119, + "model": "base.region", + "fields": { + "rght": 376, + "code": "IMN", + "name": "Isle of Man", + "parent": 5, + "level": 2, + "lft": 375, + "tree_id": 90, + "bbox_x0": -4.83018, + "bbox_x1": -4.31007, + "bbox_y0": 54.04464, + "bbox_y1": 54.418839 + } + }, + { + "pk": 120, + "model": "base.region", + "fields": { + "rght": 442, + "code": "ISR", + "name": "Israel", + "parent": 15, + "level": 2, + "lft": 441, + "tree_id": 90, + "bbox_x0": 34.2677, + "bbox_x1": 35.940941, + "bbox_y0": 29.4965, + "bbox_y1": 33.43338 + } + }, + { + "pk": 121, + "model": "base.region", + "fields": { + "rght": 378, + "code": "ITA", + "name": "Italy", + "parent": 5, + "level": 2, + "lft": 377, + "tree_id": 90, + "bbox_x0": 6.62665, + "bbox_x1": 18.520281, + "bbox_y0": 35.49308, + "bbox_y1": 47.091999 + } + }, + { + "pk": 122, + "model": "base.region", + "fields": { + "rght": 159, + "code": "JAM", + "name": "Jamaica", + "parent": 255, + "level": 3, + "lft": 158, + "tree_id": 90, + "bbox_x0": -78.366638, + "bbox_x1": -75.982857, + "bbox_y0": 16.949551, + "bbox_y1": 18.52697 + } + }, + { + "pk": 123, + "model": "base.region", + "fields": { + "rght": 271, + "code": "JPN", + "name": "Japan", + "parent": 258, + "level": 3, + "lft": 270, + "tree_id": 90, + "bbox_x0": 122.933647, + "bbox_x1": 153.986847, + "bbox_y0": 20.4251, + "bbox_y1": 45.557709 + } + }, + { + "pk": 124, + "model": "base.region", + "fields": { + "rght": 380, + "code": "JEY", + "name": "Jersey", + "parent": 5, + "level": 2, + "lft": 379, + "tree_id": 90, + "bbox_x0": -2.25505, + "bbox_x1": -2.01129, + "bbox_y0": 49.16209, + "bbox_y1": 49.26231 + } + }, + { + "pk": 125, + "model": "base.region", + "fields": { + "rght": 444, + "code": "JOR", + "name": "Jordan", + "parent": 15, + "level": 2, + "lft": 443, + "tree_id": 90, + "bbox_x0": 34.960232, + "bbox_x1": 39.301128, + "bbox_y0": 29.18409, + "bbox_y1": 33.374828 + } + }, + { + "pk": 126, + "model": "base.region", + "fields": { + "rght": 251, + "code": "KAZ", + "name": "Kazakhstan", + "parent": 8, + "level": 3, + "lft": 250, + "tree_id": 90, + "bbox_x0": 46.491859, + "bbox_x1": 87.312737, + "bbox_y0": 40.566689, + "bbox_y1": 55.431808 + } + }, + { + "pk": 127, + "model": "base.region", + "fields": { + "rght": 25, + "code": "KEN", + "name": "Kenya", + "parent": 12, + "level": 3, + "lft": 24, + "tree_id": 90, + "bbox_x0": 33.90884, + "bbox_x1": 41.899059, + "bbox_y0": -4.71712, + "bbox_y1": 4.62933 + } + }, + { + "pk": 128, + "model": "base.region", + "fields": { + "rght": 478, + "code": "KIR", + "name": "Kiribati", + "parent": 256, + "level": 2, + "lft": 477, + "tree_id": 90, + "bbox_x0": 158.418335, + "bbox_x1": -150.208359, + "bbox_y0": -11.43703, + "bbox_y1": 4.71956 + } + }, + { + "pk": 129, + "model": "base.region", + "fields": { + "rght": 446, + "code": "KWT", + "name": "Kuwait", + "parent": 15, + "level": 2, + "lft": 445, + "tree_id": 90, + "bbox_x0": 46.55751, + "bbox_x1": 48.78384, + "bbox_y0": 28.5245, + "bbox_y1": 30.0958 + } + }, + { + "pk": 130, + "model": "base.region", + "fields": { + "rght": 253, + "code": "KGZ", + "name": "Kyrgyzstan", + "parent": 8, + "level": 3, + "lft": 252, + "tree_id": 90, + "bbox_x0": 69.276619, + "bbox_x1": 80.28318, + "bbox_y0": 39.17284, + "bbox_y1": 43.238239 + } + }, + { + "pk": 131, + "model": "base.region", + "fields": { + "rght": 301, + "code": "LAO", + "name": "Lao People's Democratic Republic", + "parent": 7, + "level": 3, + "lft": 300, + "tree_id": 90, + "bbox_x0": 100.093048, + "bbox_x1": 107.697021, + "bbox_y0": 13.91002, + "bbox_y1": 22.500389 + } + }, + { + "pk": 132, + "model": "base.region", + "fields": { + "rght": 382, + "code": "LVA", + "name": "Latvia", + "parent": 5, + "level": 2, + "lft": 381, + "tree_id": 90, + "bbox_x0": 20.966061, + "bbox_x1": 28.244431, + "bbox_y0": 55.67276, + "bbox_y1": 58.087448 + } + }, + { + "pk": 133, + "model": "base.region", + "fields": { + "rght": 448, + "code": "LBN", + "name": "Lebanon", + "parent": 15, + "level": 2, + "lft": 447, + "tree_id": 90, + "bbox_x0": 35.10368, + "bbox_x1": 36.622791, + "bbox_y0": 33.048908, + "bbox_y1": 34.69268 + } + }, + { + "pk": 134, + "model": "base.region", + "fields": { + "rght": 69, + "code": "LSO", + "name": "Lesotho", + "parent": 14, + "level": 3, + "lft": 68, + "tree_id": 90, + "bbox_x0": 27.02906, + "bbox_x1": 29.465771, + "bbox_y0": -30.668961, + "bbox_y1": -28.57205 + } + }, + { + "pk": 135, + "model": "base.region", + "fields": { + "rght": 109, + "code": "LBR", + "name": "Liberia", + "parent": 13, + "level": 3, + "lft": 108, + "tree_id": 90, + "bbox_x0": -11.49208, + "bbox_x1": -7.36511, + "bbox_y0": 4.35305, + "bbox_y1": 8.55179 + } + }, + { + "pk": 136, + "model": "base.region", + "fields": { + "rght": 55, + "code": "LBY", + "name": "Libyan Arab Jamahiriya", + "parent": 11, + "level": 3, + "lft": 54, + "tree_id": 90, + "bbox_x0": 9.38702, + "bbox_x1": 25.15061, + "bbox_y0": 19.508039, + "bbox_y1": 33.168999 + } + }, + { + "pk": 137, + "model": "base.region", + "fields": { + "rght": 384, + "code": "LIE", + "name": "Liechtenstein", + "parent": 5, + "level": 2, + "lft": 383, + "tree_id": 90, + "bbox_x0": 9.47181, + "bbox_x1": 9.63578, + "bbox_y0": 47.04834, + "bbox_y1": 47.270649 + } + }, + { + "pk": 138, + "model": "base.region", + "fields": { + "rght": 386, + "code": "LTU", + "name": "Lithuania", + "parent": 5, + "level": 2, + "lft": 385, + "tree_id": 90, + "bbox_x0": 20.953199, + "bbox_x1": 26.835581, + "bbox_y0": 53.89748, + "bbox_y1": 56.450432 + } + }, + { + "pk": 139, + "model": "base.region", + "fields": { + "rght": 388, + "code": "LUX", + "name": "Luxembourg", + "parent": 5, + "level": 2, + "lft": 387, + "tree_id": 90, + "bbox_x0": 5.73579, + "bbox_x1": 6.53117, + "bbox_y0": 49.447689, + "bbox_y1": 50.182751 + } + }, + { + "pk": 140, + "model": "base.region", + "fields": { + "rght": 390, + "code": "MKD", + "name": "Macedonia", + "parent": 5, + "level": 2, + "lft": 389, + "tree_id": 90, + "bbox_x0": 20.4645, + "bbox_x1": 23.038071, + "bbox_y0": 40.860111, + "bbox_y1": 42.345501 + } + }, + { + "pk": 141, + "model": "base.region", + "fields": { + "rght": 27, + "code": "MDG", + "name": "Madagascar", + "parent": 12, + "level": 3, + "lft": 26, + "tree_id": 90, + "bbox_x0": 43.19138, + "bbox_x1": 50.483799, + "bbox_y0": -25.60894, + "bbox_y1": -11.94543 + } + }, + { + "pk": 142, + "model": "base.region", + "fields": { + "rght": 29, + "code": "MWI", + "name": "Malawi", + "parent": 12, + "level": 3, + "lft": 28, + "tree_id": 90, + "bbox_x0": 32.668991, + "bbox_x1": 35.920441, + "bbox_y0": -17.129459, + "bbox_y1": -9.36468 + } + }, + { + "pk": 143, + "model": "base.region", + "fields": { + "rght": 303, + "code": "MYS", + "name": "Malaysia", + "parent": 7, + "level": 3, + "lft": 302, + "tree_id": 90, + "bbox_x0": 98.935059, + "bbox_x1": 119.448433, + "bbox_y0": 0.66364, + "bbox_y1": 7.58378 + } + }, + { + "pk": 144, + "model": "base.region", + "fields": { + "rght": 285, + "code": "MDV", + "name": "Maldives", + "parent": 9, + "level": 3, + "lft": 284, + "tree_id": 90, + "bbox_x0": 72.616219, + "bbox_x1": 73.76712, + "bbox_y0": -2.90045, + "bbox_y1": 7.11712 + } + }, + { + "pk": 145, + "model": "base.region", + "fields": { + "rght": 111, + "code": "MLI", + "name": "Mali", + "parent": 13, + "level": 3, + "lft": 110, + "tree_id": 90, + "bbox_x0": -12.24261, + "bbox_x1": 4.24495, + "bbox_y0": 10.15951, + "bbox_y1": 25 + } + }, + { + "pk": 146, + "model": "base.region", + "fields": { + "rght": 392, + "code": "MLT", + "name": "Malta", + "parent": 5, + "level": 2, + "lft": 391, + "tree_id": 90, + "bbox_x0": 14.18341, + "bbox_x1": 14.5766, + "bbox_y0": 35.786282, + "bbox_y1": 36.081821 + } + }, + { + "pk": 147, + "model": "base.region", + "fields": { + "rght": 480, + "code": "MHL", + "name": "Marshall Islands", + "parent": 256, + "level": 2, + "lft": 479, + "tree_id": 90, + "bbox_x0": 162.143265, + "bbox_x1": 172.161987, + "bbox_y0": 4.57487, + "bbox_y1": 14.65516 + } + }, + { + "pk": 148, + "model": "base.region", + "fields": { + "rght": 161, + "code": "MTQ", + "name": "Martinique", + "parent": 255, + "level": 3, + "lft": 160, + "tree_id": 90, + "bbox_x0": -61.23011, + "bbox_x1": -60.81551, + "bbox_y0": 14.38244, + "bbox_y1": 14.87881 + } + }, + { + "pk": 149, + "model": "base.region", + "fields": { + "rght": 113, + "code": "MRT", + "name": "Mauritania", + "parent": 13, + "level": 3, + "lft": 112, + "tree_id": 90, + "bbox_x0": -17.066521, + "bbox_x1": -4.8352, + "bbox_y0": 14.71554, + "bbox_y1": 27.298071 + } + }, + { + "pk": 150, + "model": "base.region", + "fields": { + "rght": 31, + "code": "MUS", + "name": "Mauritius", + "parent": 12, + "level": 3, + "lft": 30, + "tree_id": 90, + "bbox_x0": 56.512711, + "bbox_x1": 63.525379, + "bbox_y0": -20.525709, + "bbox_y1": -10.31925 + } + }, + { + "pk": 151, + "model": "base.region", + "fields": { + "rght": 33, + "code": "MYT", + "name": "Mayotte", + "parent": 12, + "level": 3, + "lft": 32, + "tree_id": 90, + "bbox_x0": 45.01461, + "bbox_x1": 45.317131, + "bbox_y0": -13.00045, + "bbox_y1": -12.63383 + } + }, + { + "pk": 152, + "model": "base.region", + "fields": { + "rght": 209, + "code": "MEX", + "name": "Mexico", + "parent": 2, + "level": 3, + "lft": 208, + "tree_id": 90, + "bbox_x0": -118.867172, + "bbox_x1": -86.703392, + "bbox_y0": 14.53285, + "bbox_y1": 32.71862 + } + }, + { + "pk": 153, + "model": "base.region", + "fields": { + "rght": 482, + "code": "FSM", + "name": "Micronesia, Federated States of", + "parent": 256, + "level": 2, + "lft": 481, + "tree_id": 90, + "bbox_x0": 138.052856, + "bbox_x1": 163.034912, + "bbox_y0": 5.25984, + "bbox_y1": 10.02222 + } + }, + { + "pk": 154, + "model": "base.region", + "fields": { + "rght": 394, + "code": "MCO", + "name": "Monaco", + "parent": 5, + "level": 2, + "lft": 393, + "tree_id": 90, + "bbox_x0": 7.4091, + "bbox_x1": 7.43948, + "bbox_y0": 43.724789, + "bbox_y1": 43.7519 + } + }, + { + "pk": 155, + "model": "base.region", + "fields": { + "rght": 273, + "code": "MNG", + "name": "Mongolia", + "parent": 258, + "level": 3, + "lft": 272, + "tree_id": 90, + "bbox_x0": 87.749496, + "bbox_x1": 119.92411, + "bbox_y0": 41.567501, + "bbox_y1": 52.154099 + } + }, + { + "pk": 156, + "model": "base.region", + "fields": { + "rght": 396, + "code": "MNE", + "name": "Montenegro", + "parent": 5, + "level": 2, + "lft": 395, + "tree_id": 90, + "bbox_x0": 18.43705, + "bbox_x1": 20.41608, + "bbox_y0": 41.84808, + "bbox_y1": 43.542912 + } + }, + { + "pk": 157, + "model": "base.region", + "fields": { + "rght": 163, + "code": "MSR", + "name": "Montserrat", + "parent": 255, + "level": 3, + "lft": 162, + "tree_id": 90, + "bbox_x0": -62.24258, + "bbox_x1": -62.14642, + "bbox_y0": 16.671, + "bbox_y1": 16.81732 + } + }, + { + "pk": 158, + "model": "base.region", + "fields": { + "rght": 57, + "code": "MAR", + "name": "Morocco", + "parent": 11, + "level": 3, + "lft": 56, + "tree_id": 90, + "bbox_x0": -11.7805, + "bbox_x1": -1.02441, + "bbox_y0": 26.946489, + "bbox_y1": 35.921909 + } + }, + { + "pk": 159, + "model": "base.region", + "fields": { + "rght": 35, + "code": "MOZ", + "name": "Mozambique", + "parent": 12, + "level": 3, + "lft": 34, + "tree_id": 90, + "bbox_x0": 30.21731, + "bbox_x1": 40.844471, + "bbox_y0": -26.868679, + "bbox_y1": -10.47188 + } + }, + { + "pk": 160, + "model": "base.region", + "fields": { + "rght": 305, + "code": "MMR", + "name": "Myanmar", + "parent": 7, + "level": 3, + "lft": 304, + "tree_id": 90, + "bbox_x0": 92.189209, + "bbox_x1": 101.176788, + "bbox_y0": 9.60035, + "bbox_y1": 28.543249 + } + }, + { + "pk": 161, + "model": "base.region", + "fields": { + "rght": 71, + "code": "NMB", + "name": "Namibia", + "parent": 14, + "level": 3, + "lft": 70, + "tree_id": 90, + "bbox_x0": 11.71563, + "bbox_x1": 25.256701, + "bbox_y0": -28.97142, + "bbox_y1": -16.95989 + } + }, + { + "pk": 162, + "model": "base.region", + "fields": { + "rght": 484, + "code": "NRU", + "name": "Nauru", + "parent": 256, + "level": 2, + "lft": 483, + "tree_id": 90, + "bbox_x0": 166.899017, + "bbox_x1": 166.945267, + "bbox_y0": -0.55232, + "bbox_y1": -0.50429 + } + }, + { + "pk": 163, + "model": "base.region", + "fields": { + "rght": 287, + "code": "NPL", + "name": "Nepal", + "parent": 9, + "level": 3, + "lft": 286, + "tree_id": 90, + "bbox_x0": 80.056221, + "bbox_x1": 88.199318, + "bbox_y0": 26.356501, + "bbox_y1": 30.433001 + } + }, + { + "pk": 164, + "model": "base.region", + "fields": { + "rght": 398, + "code": "NLD", + "name": "Netherlands", + "parent": 5, + "level": 2, + "lft": 397, + "tree_id": 90, + "bbox_x0": 3.35794, + "bbox_x1": 7.2267, + "bbox_y0": 50.750401, + "bbox_y1": 53.554241 + } + }, + { + "pk": 165, + "model": "base.region", + "fields": { + "rght": 165, + "code": "NAN", + "name": "Netherlands Antilles", + "parent": 255, + "level": 3, + "lft": 164, + "tree_id": 90, + "bbox_x0": -69.157188, + "bbox_x1": -62.943661, + "bbox_y0": 11.97318, + "bbox_y1": 18.07024 + } + }, + { + "pk": 166, + "model": "base.region", + "fields": { + "rght": 486, + "code": "NCL", + "name": "New Caledonia", + "parent": 256, + "level": 2, + "lft": 485, + "tree_id": 90, + "bbox_x0": 158.332855, + "bbox_x1": 172.061417, + "bbox_y0": -22.90045, + "bbox_y1": -18.01622 + } + }, + { + "pk": 167, + "model": "base.region", + "fields": { + "rght": 488, + "code": "NZL", + "name": "New Zealand", + "parent": 256, + "level": 2, + "lft": 487, + "tree_id": 90, + "bbox_x0": 165.883804, + "bbox_x1": -175.987198, + "bbox_y0": -52.618591, + "bbox_y1": -29.20997 + } + }, + { + "pk": 168, + "model": "base.region", + "fields": { + "rght": 199, + "code": "NIC", + "name": "Nicaragua", + "parent": 3, + "level": 3, + "lft": 198, + "tree_id": 90, + "bbox_x0": -87.6903, + "bbox_x1": -82.592072, + "bbox_y0": 10.70754, + "bbox_y1": 15.0259 + } + }, + { + "pk": 169, + "model": "base.region", + "fields": { + "rght": 115, + "code": "NER", + "name": "Niger", + "parent": 13, + "level": 3, + "lft": 114, + "tree_id": 90, + "bbox_x0": 0.16625, + "bbox_x1": 15.99564, + "bbox_y0": 11.69697, + "bbox_y1": 23.525021 + } + }, + { + "pk": 170, + "model": "base.region", + "fields": { + "rght": 117, + "code": "NGA", + "name": "Nigeria", + "parent": 13, + "level": 3, + "lft": 116, + "tree_id": 90, + "bbox_x0": 2.66844, + "bbox_x1": 14.68006, + "bbox_y0": 4.27714, + "bbox_y1": 13.892 + } + }, + { + "pk": 171, + "model": "base.region", + "fields": { + "rght": 490, + "code": "NIU", + "name": "Niue", + "parent": 256, + "level": 2, + "lft": 489, + "tree_id": 90, + "bbox_x0": -169.951004, + "bbox_x1": -169.775177, + "bbox_y0": -19.152189, + "bbox_y1": -18.951059 + } + }, + { + "pk": 172, + "model": "base.region", + "fields": { + "rght": 492, + "code": "NFK", + "name": "Norfolk Island", + "parent": 256, + "level": 2, + "lft": 491, + "tree_id": 90, + "bbox_x0": 167.949493, + "bbox_x1": 168.091614, + "bbox_y0": -29.11937, + "bbox_y1": -28.99493 + } + }, + { + "pk": 173, + "model": "base.region", + "fields": { + "rght": 494, + "code": "MNP", + "name": "Northern Mariana Islands", + "parent": 256, + "level": 2, + "lft": 493, + "tree_id": 90, + "bbox_x0": 136.082855, + "bbox_x1": 146.081223, + "bbox_y0": 14.10735, + "bbox_y1": 20.41712 + } + }, + { + "pk": 174, + "model": "base.region", + "fields": { + "rght": 400, + "code": "NOR", + "name": "Norway", + "parent": 5, + "level": 2, + "lft": 399, + "tree_id": 90, + "bbox_x0": 4.43292, + "bbox_x1": 31.168409, + "bbox_y0": 57.962582, + "bbox_y1": 71.185509 + } + }, + { + "pk": 175, + "model": "base.region", + "fields": { + "rght": 450, + "code": "PSE", + "name": "Occupied Palestinian Territory", + "parent": 15, + "level": 2, + "lft": 449, + "tree_id": 90, + "bbox_x0": 34.230461, + "bbox_x1": 35.876499, + "bbox_y0": 31.223499, + "bbox_y1": 33.340099 + } + }, + { + "pk": 176, + "model": "base.region", + "fields": { + "rght": 452, + "code": "OMN", + "name": "Oman", + "parent": 15, + "level": 2, + "lft": 451, + "tree_id": 90, + "bbox_x0": 51.882011, + "bbox_x1": 59.83651, + "bbox_y0": 16.6457, + "bbox_y1": 26.50045 + } + }, + { + "pk": 177, + "model": "base.region", + "fields": { + "rght": 289, + "code": "PAK", + "name": "Pakistan", + "parent": 9, + "level": 3, + "lft": 288, + "tree_id": 90, + "bbox_x0": 60.87859, + "bbox_x1": 77.840813, + "bbox_y0": 23.786699, + "bbox_y1": 37.097 + } + }, + { + "pk": 178, + "model": "base.region", + "fields": { + "rght": 496, + "code": "PLW", + "name": "Palau", + "parent": 256, + "level": 2, + "lft": 495, + "tree_id": 90, + "bbox_x0": 131.169235, + "bbox_x1": 134.723724, + "bbox_y0": 3.00114, + "bbox_y1": 8.09291 + } + }, + { + "pk": 179, + "model": "base.region", + "fields": { + "rght": 201, + "code": "PAN", + "name": "Panama", + "parent": 3, + "level": 3, + "lft": 200, + "tree_id": 90, + "bbox_x0": -83.051453, + "bbox_x1": -77.17411, + "bbox_y0": 7.1979, + "bbox_y1": 9.65045 + } + }, + { + "pk": 180, + "model": "base.region", + "fields": { + "rght": 498, + "code": "PNG", + "name": "Papua New Guinea", + "parent": 256, + "level": 2, + "lft": 497, + "tree_id": 90, + "bbox_x0": 140.842865, + "bbox_x1": 159.48378, + "bbox_y0": -11.65785, + "bbox_y1": -0.86622 + } + }, + { + "pk": 181, + "model": "base.region", + "fields": { + "rght": 233, + "code": "PRY", + "name": "Paraguay", + "parent": 4, + "level": 3, + "lft": 232, + "tree_id": 90, + "bbox_x0": -62.647072, + "bbox_x1": -54.25935, + "bbox_y0": -27.60873, + "bbox_y1": -19.294041 + } + }, + { + "pk": 182, + "model": "base.region", + "fields": { + "rght": 235, + "code": "PER", + "name": "Peru", + "parent": 4, + "level": 3, + "lft": 234, + "tree_id": 90, + "bbox_x0": -81.326736, + "bbox_x1": -68.677979, + "bbox_y0": -18.34972, + "bbox_y1": -0.01297 + } + }, + { + "pk": 183, + "model": "base.region", + "fields": { + "rght": 307, + "code": "PHL", + "name": "Philippines", + "parent": 7, + "level": 3, + "lft": 306, + "tree_id": 90, + "bbox_x0": 116.812721, + "bbox_x1": 126.856628, + "bbox_y0": 4.46811, + "bbox_y1": 21.23415 + } + }, + { + "pk": 184, + "model": "base.region", + "fields": { + "rght": 500, + "code": "PCN", + "name": "Pitcairn", + "parent": 256, + "level": 2, + "lft": 499, + "tree_id": 90, + "bbox_x0": -130.746033, + "bbox_x1": -124.772842, + "bbox_y0": -25.07752, + "bbox_y1": -23.917271 + } + }, + { + "pk": 185, + "model": "base.region", + "fields": { + "rght": 402, + "code": "POL", + "name": "Poland", + "parent": 5, + "level": 2, + "lft": 401, + "tree_id": 90, + "bbox_x0": 14.12281, + "bbox_x1": 24.145781, + "bbox_y0": 49.002022, + "bbox_y1": 54.835812 + } + }, + { + "pk": 186, + "model": "base.region", + "fields": { + "rght": 404, + "code": "PRT", + "name": "Portugal", + "parent": 5, + "level": 2, + "lft": 403, + "tree_id": 90, + "bbox_x0": -31.266001, + "bbox_x1": -6.18931, + "bbox_y0": 30.028061, + "bbox_y1": 42.154121 + } + }, + { + "pk": 187, + "model": "base.region", + "fields": { + "rght": 167, + "code": "PRI", + "name": "Puerto Rico", + "parent": 255, + "level": 3, + "lft": 166, + "tree_id": 90, + "bbox_x0": -67.942719, + "bbox_x1": -65.219978, + "bbox_y0": 17.883039, + "bbox_y1": 18.520161 + } + }, + { + "pk": 188, + "model": "base.region", + "fields": { + "rght": 454, + "code": "QAT", + "name": "Qatar", + "parent": 15, + "level": 2, + "lft": 453, + "tree_id": 90, + "bbox_x0": 50.757011, + "bbox_x1": 52.427509, + "bbox_y0": 24.482901, + "bbox_y1": 26.177509 + } + }, + { + "pk": 189, + "model": "base.region", + "fields": { + "rght": 275, + "code": "KOR", + "name": "Republic of Korea", + "parent": 258, + "level": 3, + "lft": 274, + "tree_id": 90, + "bbox_x0": 124.608147, + "bbox_x1": 130.933899, + "bbox_y0": 33.10611, + "bbox_y1": 38.612301 + } + }, + { + "pk": 190, + "model": "base.region", + "fields": { + "rght": 406, + "code": "MDA", + "name": "Republic of Moldova", + "parent": 5, + "level": 2, + "lft": 405, + "tree_id": 90, + "bbox_x0": 26.618879, + "bbox_x1": 30.16374, + "bbox_y0": 45.468498, + "bbox_y1": 48.490162 + } + }, + { + "pk": 191, + "model": "base.region", + "fields": { + "rght": 37, + "code": "REU", + "name": "Reunion", + "parent": 12, + "level": 3, + "lft": 36, + "tree_id": 90, + "bbox_x0": 55.219082, + "bbox_x1": 55.845039, + "bbox_y0": -21.37221, + "bbox_y1": -20.85685 + } + }, + { + "pk": 192, + "model": "base.region", + "fields": { + "rght": 408, + "code": "ROU", + "name": "Romania", + "parent": 5, + "level": 2, + "lft": 407, + "tree_id": 90, + "bbox_x0": 20.269791, + "bbox_x1": 29.691, + "bbox_y0": 43.626999, + "bbox_y1": 48.266891 + } + }, + { + "pk": 193, + "model": "base.region", + "fields": { + "rght": 410, + "code": "RUS", + "name": "Russian Federation", + "parent": 5, + "level": 2, + "lft": 409, + "tree_id": 90, + "bbox_x0": 19.638861, + "bbox_x1": 180, + "bbox_y0": 41.185902, + "bbox_y1": 81.856903 + } + }, + { + "pk": 194, + "model": "base.region", + "fields": { + "rght": 39, + "code": "RWA", + "name": "Rwanda", + "parent": 12, + "level": 3, + "lft": 38, + "tree_id": 90, + "bbox_x0": 28.8568, + "bbox_x1": 30.89596, + "bbox_y0": -2.84067, + "bbox_y1": -1.05348 + } + }, + { + "pk": 195, + "model": "base.region", + "fields": { + "rght": 73, + "code": "SHN", + "name": "Saint Helena", + "parent": 14, + "level": 3, + "lft": 72, + "tree_id": 90, + "bbox_x0": -14.44153, + "bbox_x1": -5.63286, + "bbox_y0": -40.400452, + "bbox_y1": -7.87757 + } + }, + { + "pk": 196, + "model": "base.region", + "fields": { + "rght": 169, + "code": "KNA", + "name": "Saint Kitts and Nevis", + "parent": 255, + "level": 3, + "lft": 168, + "tree_id": 90, + "bbox_x0": -62.86956, + "bbox_x1": -62.543259, + "bbox_y0": 17.095341, + "bbox_y1": 17.420111 + } + }, + { + "pk": 197, + "model": "base.region", + "fields": { + "rght": 171, + "code": "LCA", + "name": "Saint Lucia", + "parent": 255, + "level": 3, + "lft": 170, + "tree_id": 90, + "bbox_x0": -61.07415, + "bbox_x1": -60.866051, + "bbox_y0": 13.70477, + "bbox_y1": 14.10324 + } + }, + { + "pk": 198, + "model": "base.region", + "fields": { + "rght": 173, + "code": "SPM", + "name": "Saint Pierre and Miquelon", + "parent": 255, + "level": 3, + "lft": 172, + "tree_id": 90, + "bbox_x0": -56.420658, + "bbox_x1": -56.157372, + "bbox_y0": 46.753269, + "bbox_y1": 47.14629 + } + }, + { + "pk": 199, + "model": "base.region", + "fields": { + "rght": 175, + "code": "VCT", + "name": "Saint Vincent and the Grenadines", + "parent": 255, + "level": 3, + "lft": 174, + "tree_id": 90, + "bbox_x0": -61.459251, + "bbox_x1": -61.11388, + "bbox_y0": 12.58101, + "bbox_y1": 13.37783 + } + }, + { + "pk": 200, + "model": "base.region", + "fields": { + "rght": 177, + "code": "BLM", + "name": "Saint-Barthelemy", + "parent": 255, + "level": 3, + "lft": 176, + "tree_id": 90, + "bbox_x0": -62.9338, + "bbox_x1": -62.78286, + "bbox_y0": 17.86622, + "bbox_y1": 17.968599 + } + }, + { + "pk": 201, + "model": "base.region", + "fields": { + "rght": 179, + "code": "MAF", + "name": "Saint-Martin (French part)", + "parent": 255, + "level": 3, + "lft": 178, + "tree_id": 90, + "bbox_x0": -63.15276, + "bbox_x1": -62.972778, + "bbox_y0": 18.052231, + "bbox_y1": 18.13069 + } + }, + { + "pk": 202, + "model": "base.region", + "fields": { + "rght": 502, + "code": "WSM", + "name": "Samoa", + "parent": 256, + "level": 2, + "lft": 501, + "tree_id": 90, + "bbox_x0": -172.798584, + "bbox_x1": -171.33287, + "bbox_y0": -14.06353, + "bbox_y1": -13.4322 + } + }, + { + "pk": 203, + "model": "base.region", + "fields": { + "rght": 412, + "code": "SMR", + "name": "San Marino", + "parent": 5, + "level": 2, + "lft": 411, + "tree_id": 90, + "bbox_x0": 12.40306, + "bbox_x1": 12.51651, + "bbox_y0": 43.893299, + "bbox_y1": 43.992168 + } + }, + { + "pk": 204, + "model": "base.region", + "fields": { + "rght": 119, + "code": "STP", + "name": "Sao Tome and Principe", + "parent": 13, + "level": 3, + "lft": 118, + "tree_id": 90, + "bbox_x0": 5.59955, + "bbox_x1": 7.46637, + "bbox_y0": -0.014, + "bbox_y1": 1.73378 + } + }, + { + "pk": 205, + "model": "base.region", + "fields": { + "rght": 456, + "code": "SAU", + "name": "Saudi Arabia", + "parent": 15, + "level": 2, + "lft": 455, + "tree_id": 90, + "bbox_x0": 34.508282, + "bbox_x1": 55.666672, + "bbox_y0": 16.261169, + "bbox_y1": 32.173481 + } + }, + { + "pk": 206, + "model": "base.region", + "fields": { + "rght": 121, + "code": "SEN", + "name": "Senegal", + "parent": 13, + "level": 3, + "lft": 120, + "tree_id": 90, + "bbox_x0": -17.535231, + "bbox_x1": -11.35588, + "bbox_y0": 12.30727, + "bbox_y1": 16.691629 + } + }, + { + "pk": 207, + "model": "base.region", + "fields": { + "rght": 414, + "code": "SRB", + "name": "Serbia", + "parent": 5, + "level": 2, + "lft": 413, + "tree_id": 90, + "bbox_x0": 18.814581, + "bbox_x1": 23.007, + "bbox_y0": 41.908611, + "bbox_y1": 46.191002 + } + }, + { + "pk": 208, + "model": "base.region", + "fields": { + "rght": 41, + "code": "SYC", + "name": "Seychelles", + "parent": 12, + "level": 3, + "lft": 40, + "tree_id": 90, + "bbox_x0": 46.199211, + "bbox_x1": 56.279499, + "bbox_y0": -10.21712, + "bbox_y1": -3.71151 + } + }, + { + "pk": 209, + "model": "base.region", + "fields": { + "rght": 123, + "code": "SLE", + "name": "Sierra Leone", + "parent": 13, + "level": 3, + "lft": 122, + "tree_id": 90, + "bbox_x0": -13.30763, + "bbox_x1": -10.28423, + "bbox_y0": 6.92868, + "bbox_y1": 10.00043 + } + }, + { + "pk": 210, + "model": "base.region", + "fields": { + "rght": 309, + "code": "SGP", + "name": "Singapore", + "parent": 7, + "level": 3, + "lft": 308, + "tree_id": 90, + "bbox_x0": 103.618248, + "bbox_x1": 104.40847, + "bbox_y0": 1.1158, + "bbox_y1": 1.47062 + } + }, + { + "pk": 211, + "model": "base.region", + "fields": { + "rght": 416, + "code": "SVK", + "name": "Slovakia", + "parent": 5, + "level": 2, + "lft": 415, + "tree_id": 90, + "bbox_x0": 16.833179, + "bbox_x1": 22.570299, + "bbox_y0": 47.728001, + "bbox_y1": 49.603001 + } + }, + { + "pk": 212, + "model": "base.region", + "fields": { + "rght": 418, + "code": "SVN", + "name": "Slovenia", + "parent": 5, + "level": 2, + "lft": 417, + "tree_id": 90, + "bbox_x0": 13.37551, + "bbox_x1": 16.59656, + "bbox_y0": 45.416, + "bbox_y1": 46.87788 + } + }, + { + "pk": 213, + "model": "base.region", + "fields": { + "rght": 504, + "code": "SLB", + "name": "Solomon Islands", + "parent": 256, + "level": 2, + "lft": 503, + "tree_id": 90, + "bbox_x0": 155.508667, + "bbox_x1": 170.200455, + "bbox_y0": -12.2919, + "bbox_y1": -5.16622 + } + }, + { + "pk": 214, + "model": "base.region", + "fields": { + "rght": 43, + "code": "SOM", + "name": "Somalia", + "parent": 12, + "level": 3, + "lft": 42, + "tree_id": 90, + "bbox_x0": 40.988628, + "bbox_x1": 51.413029, + "bbox_y0": -1.66205, + "bbox_y1": 11.9852 + } + }, + { + "pk": 215, + "model": "base.region", + "fields": { + "rght": 75, + "code": "ZAF", + "name": "South Africa", + "parent": 14, + "level": 3, + "lft": 74, + "tree_id": 90, + "bbox_x0": 16.46841, + "bbox_x1": 37.993172, + "bbox_y0": -46.990009, + "bbox_y1": -22.12472 + } + }, + { + "pk": 216, + "model": "base.region", + "fields": { + "rght": 420, + "code": "ESP", + "name": "Spain", + "parent": 5, + "level": 2, + "lft": 419, + "tree_id": 90, + "bbox_x0": -18.160789, + "bbox_x1": 4.32788, + "bbox_y0": 27.63546, + "bbox_y1": 43.789959 + } + }, + { + "pk": 217, + "model": "base.region", + "fields": { + "rght": 291, + "code": "LKA", + "name": "Sri Lanka", + "parent": 9, + "level": 3, + "lft": 290, + "tree_id": 90, + "bbox_x0": 79.516212, + "bbox_x1": 81.88121, + "bbox_y0": 5.9167, + "bbox_y1": 9.8312 + } + }, + { + "pk": 218, + "model": "base.region", + "fields": { + "rght": 59, + "code": "SDN", + "name": "Sudan", + "parent": 11, + "level": 3, + "lft": 58, + "tree_id": 90, + "bbox_x0": 21.83894, + "bbox_x1": 38.833801, + "bbox_y0": 3.48639, + "bbox_y1": 23.146891 + } + }, + { + "pk": 219, + "model": "base.region", + "fields": { + "rght": 237, + "code": "SUR", + "name": "Suriname", + "parent": 4, + "level": 3, + "lft": 236, + "tree_id": 90, + "bbox_x0": -58.086559, + "bbox_x1": -53.977489, + "bbox_y0": 1.83114, + "bbox_y1": 6.00454 + } + }, + { + "pk": 220, + "model": "base.region", + "fields": { + "rght": 422, + "code": "SJM", + "name": "Svalbard and Jan Mayen Islands", + "parent": 5, + "level": 2, + "lft": 421, + "tree_id": 90, + "bbox_x0": -9.07989, + "bbox_x1": 36.815269, + "bbox_y0": 70.82737, + "bbox_y1": 80.834061 + } + }, + { + "pk": 221, + "model": "base.region", + "fields": { + "rght": 77, + "code": "SWZ", + "name": "Swaziland", + "parent": 14, + "level": 3, + "lft": 76, + "tree_id": 90, + "bbox_x0": 30.7941, + "bbox_x1": 32.137272, + "bbox_y0": -27.317101, + "bbox_y1": -25.719641 + } + }, + { + "pk": 222, + "model": "base.region", + "fields": { + "rght": 424, + "code": "SWE", + "name": "Sweden", + "parent": 5, + "level": 2, + "lft": 423, + "tree_id": 90, + "bbox_x0": 10.9661, + "bbox_x1": 24.16634, + "bbox_y0": 55.33696, + "bbox_y1": 69.059937 + } + }, + { + "pk": 223, + "model": "base.region", + "fields": { + "rght": 426, + "code": "CHE", + "name": "Switzerland", + "parent": 5, + "level": 2, + "lft": 425, + "tree_id": 90, + "bbox_x0": 5.95587, + "bbox_x1": 10.49203, + "bbox_y0": 45.81802, + "bbox_y1": 47.80838 + } + }, + { + "pk": 224, + "model": "base.region", + "fields": { + "rght": 458, + "code": "SYR", + "name": "Syrian Arab Republic", + "parent": 15, + "level": 2, + "lft": 457, + "tree_id": 90, + "bbox_x0": 35.727001, + "bbox_x1": 42.384998, + "bbox_y0": 32.3106, + "bbox_y1": 37.319 + } + }, + { + "pk": 225, + "model": "base.region", + "fields": { + "rght": 255, + "code": "TJK", + "name": "Tajikistan", + "parent": 8, + "level": 3, + "lft": 254, + "tree_id": 90, + "bbox_x0": 67.387131, + "bbox_x1": 75.137222, + "bbox_y0": 36.674141, + "bbox_y1": 41.04224 + } + }, + { + "pk": 226, + "model": "base.region", + "fields": { + "rght": 311, + "code": "THA", + "name": "Thailand", + "parent": 7, + "level": 3, + "lft": 310, + "tree_id": 90, + "bbox_x0": 97.343964, + "bbox_x1": 105.636917, + "bbox_y0": 5.61257, + "bbox_y1": 20.464701 + } + }, + { + "pk": 227, + "model": "base.region", + "fields": { + "rght": 313, + "code": "TLS", + "name": "Timor-Leste", + "parent": 7, + "level": 3, + "lft": 312, + "tree_id": 90, + "bbox_x0": 124.075439, + "bbox_x1": 127.345337, + "bbox_y0": -9.51337, + "bbox_y1": -8.13741 + } + }, + { + "pk": 228, + "model": "base.region", + "fields": { + "rght": 125, + "code": "TGO", + "name": "Togo", + "parent": 13, + "level": 3, + "lft": 124, + "tree_id": 90, + "bbox_x0": -0.14731, + "bbox_x1": 1.80669, + "bbox_y0": 6.10441, + "bbox_y1": 11.13897 + } + }, + { + "pk": 229, + "model": "base.region", + "fields": { + "rght": 506, + "code": "TKL", + "name": "Tokelau", + "parent": 256, + "level": 2, + "lft": 505, + "tree_id": 90, + "bbox_x0": -172.517136, + "bbox_x1": -171.182083, + "bbox_y0": -9.43378, + "bbox_y1": -8.53288 + } + }, + { + "pk": 230, + "model": "base.region", + "fields": { + "rght": 508, + "code": "TON", + "name": "Tonga", + "parent": 256, + "level": 2, + "lft": 507, + "tree_id": 90, + "bbox_x0": -176.212646, + "bbox_x1": -173.702438, + "bbox_y0": -22.345711, + "bbox_y1": -15.55326 + } + }, + { + "pk": 231, + "model": "base.region", + "fields": { + "rght": 181, + "code": "TTO", + "name": "Trinidad and Tobago", + "parent": 255, + "level": 3, + "lft": 180, + "tree_id": 90, + "bbox_x0": -61.927391, + "bbox_x1": -60.49144, + "bbox_y0": 10.03648, + "bbox_y1": 11.36118 + } + }, + { + "pk": 232, + "model": "base.region", + "fields": { + "rght": 61, + "code": "TUN", + "name": "Tunisia", + "parent": 11, + "level": 3, + "lft": 60, + "tree_id": 90, + "bbox_x0": 7.52481, + "bbox_x1": 11.59827, + "bbox_y0": 30.240431, + "bbox_y1": 37.56712 + } + }, + { + "pk": 233, + "model": "base.region", + "fields": { + "rght": 428, + "code": "TUR", + "name": "Turkey", + "parent": 5, + "level": 2, + "lft": 427, + "tree_id": 90, + "bbox_x0": 25.664101, + "bbox_x1": 44.8297, + "bbox_y0": 35.809971, + "bbox_y1": 42.105499 + } + }, + { + "pk": 234, + "model": "base.region", + "fields": { + "rght": 257, + "code": "TKM", + "name": "Turkmenistan", + "parent": 8, + "level": 3, + "lft": 256, + "tree_id": 90, + "bbox_x0": 52.441429, + "bbox_x1": 66.684303, + "bbox_y0": 35.14109, + "bbox_y1": 42.795551 + } + }, + { + "pk": 235, + "model": "base.region", + "fields": { + "rght": 183, + "code": "TCA", + "name": "Turks and Caicos Islands", + "parent": 255, + "level": 3, + "lft": 182, + "tree_id": 90, + "bbox_x0": -72.483879, + "bbox_x1": -71.08033, + "bbox_y0": 21.170031, + "bbox_y1": 21.97361 + } + }, + { + "pk": 236, + "model": "base.region", + "fields": { + "rght": 510, + "code": "TUV", + "name": "Tuvalu", + "parent": 256, + "level": 2, + "lft": 509, + "tree_id": 90, + "bbox_x0": 176.06488, + "bbox_x1": 179.883789, + "bbox_y0": -10.75045, + "bbox_y1": -5.64198 + } + }, + { + "pk": 237, + "model": "base.region", + "fields": { + "rght": 45, + "code": "UGA", + "name": "Uganda", + "parent": 12, + "level": 3, + "lft": 44, + "tree_id": 90, + "bbox_x0": 29.573549, + "bbox_x1": 35.001251, + "bbox_y0": -1.47849, + "bbox_y1": 4.23403 + } + }, + { + "pk": 238, + "model": "base.region", + "fields": { + "rght": 430, + "code": "UKR", + "name": "Ukraine", + "parent": 5, + "level": 2, + "lft": 429, + "tree_id": 90, + "bbox_x0": 22.128811, + "bbox_x1": 40.218079, + "bbox_y0": 44.390411, + "bbox_y1": 52.375359 + } + }, + { + "pk": 239, + "model": "base.region", + "fields": { + "rght": 460, + "code": "ARE", + "name": "United Arab Emirates", + "parent": 15, + "level": 2, + "lft": 459, + "tree_id": 90, + "bbox_x0": 51.497978, + "bbox_x1": 56.38343, + "bbox_y0": 22.644409, + "bbox_y1": 26.28219 + } + }, + { + "pk": 240, + "model": "base.region", + "fields": { + "rght": 432, + "code": "GBR", + "name": "United Kingdom", + "parent": 5, + "level": 2, + "lft": 431, + "tree_id": 90, + "bbox_x0": -13.41393, + "bbox_x1": 1.76896, + "bbox_y0": 49.16209, + "bbox_y1": 60.854691 + } + }, + { + "pk": 241, + "model": "base.region", + "fields": { + "rght": 47, + "code": "TZA", + "name": "United Republic of Tanzania", + "parent": 12, + "level": 3, + "lft": 46, + "tree_id": 90, + "bbox_x0": 29.32716, + "bbox_x1": 40.443218, + "bbox_y0": -11.74569, + "bbox_y1": -0.99073 + } + }, + { + "pk": 242, + "model": "base.region", + "fields": { + "rght": 185, + "code": "VIR", + "name": "United States Virgin Islands", + "parent": 255, + "level": 3, + "lft": 184, + "tree_id": 90, + "bbox_x0": -65.086281, + "bbox_x1": -64.56517, + "bbox_y0": 17.681721, + "bbox_y1": 18.458139 + } + }, + { + "pk": 243, + "model": "base.region", + "fields": { + "rght": 211, + "code": "USA", + "name": "United States of America", + "parent": 2, + "level": 3, + "lft": 210, + "tree_id": 90, + "bbox_x0": -179.150558, + "bbox_x1": -66.940643, + "bbox_y0": 18.91172, + "bbox_y1": 71.441048 + } + }, + { + "pk": 244, + "model": "base.region", + "fields": { + "rght": 239, + "code": "URY", + "name": "Uruguay", + "parent": 4, + "level": 3, + "lft": 238, + "tree_id": 90, + "bbox_x0": -58.442719, + "bbox_x1": -53.073929, + "bbox_y0": -35.047939, + "bbox_y1": -30.08222 + } + }, + { + "pk": 245, + "model": "base.region", + "fields": { + "rght": 259, + "code": "UZB", + "name": "Uzbekistan", + "parent": 8, + "level": 3, + "lft": 258, + "tree_id": 90, + "bbox_x0": 55.996632, + "bbox_x1": 73.132271, + "bbox_y0": 37.18433, + "bbox_y1": 45.60519 + } + }, + { + "pk": 246, + "model": "base.region", + "fields": { + "rght": 512, + "code": "VUT", + "name": "Vanuatu", + "parent": 256, + "level": 2, + "lft": 511, + "tree_id": 90, + "bbox_x0": 166.524994, + "bbox_x1": 170.234802, + "bbox_y0": -20.25045, + "bbox_y1": -13.07345 + } + }, + { + "pk": 247, + "model": "base.region", + "fields": { + "rght": 241, + "code": "VEN", + "name": "Venezuela (Bolivarian Republic of)", + "parent": 4, + "level": 3, + "lft": 240, + "tree_id": 90, + "bbox_x0": -73.374313, + "bbox_x1": -59.803768, + "bbox_y0": 0.74368, + "bbox_y1": 12.2019 + } + }, + { + "pk": 248, + "model": "base.region", + "fields": { + "rght": 315, + "code": "VNM", + "name": "Viet Nam", + "parent": 7, + "level": 3, + "lft": 314, + "tree_id": 90, + "bbox_x0": 102.144592, + "bbox_x1": 116.521233, + "bbox_y0": 7.3116, + "bbox_y1": 23.39274 + } + }, + { + "pk": 249, + "model": "base.region", + "fields": { + "rght": 514, + "code": "WLF", + "name": "Wallis and Futuna Islands", + "parent": 256, + "level": 2, + "lft": 513, + "tree_id": 90, + "bbox_x0": -178.206787, + "bbox_x1": -176.08287, + "bbox_y0": -14.38779, + "bbox_y1": -13.17343 + } + }, + { + "pk": 250, + "model": "base.region", + "fields": { + "rght": 63, + "code": "ESH", + "name": "Western Sahara", + "parent": 11, + "level": 3, + "lft": 62, + "tree_id": 90, + "bbox_x0": -17.10317, + "bbox_x1": -8.66942, + "bbox_y0": 20.774151, + "bbox_y1": 28.219179 + } + }, + { + "pk": 251, + "model": "base.region", + "fields": { + "rght": 462, + "code": "YEM", + "name": "Yemen", + "parent": 15, + "level": 2, + "lft": 461, + "tree_id": 90, + "bbox_x0": 41.809608, + "bbox_x1": 54.535992, + "bbox_y0": 12.10717, + "bbox_y1": 19.00276 + } + }, + { + "pk": 252, + "model": "base.region", + "fields": { + "rght": 79, + "code": "ZMB", + "name": "Zambia", + "parent": 14, + "level": 3, + "lft": 78, + "tree_id": 90, + "bbox_x0": 21.99938, + "bbox_x1": 33.705711, + "bbox_y0": -18.07947, + "bbox_y1": -8.22436 + } + }, + { + "pk": 253, + "model": "base.region", + "fields": { + "rght": 81, + "code": "ZWE", + "name": "Zimbabwe", + "parent": 14, + "level": 3, + "lft": 80, + "tree_id": 90, + "bbox_x0": 25.23702, + "bbox_x1": 33.056301, + "bbox_y0": -22.41773, + "bbox_y1": -15.60883 + } + }, + { + "pk": 254, + "model": "base.region", + "fields": { + "rght": 243, + "code": "AME", + "name": "Americas", + "parent": null, + "level": 1, + "lft": 128, + "tree_id": 90, + "bbox_x0": -84.114449, + "bbox_x1": -84.097572, + "bbox_y0": 9.9313, + "bbox_y1": 9.94792 + } + }, + { + "pk": 255, + "model": "base.region", + "fields": { + "rght": 186, + "code": "CRB", + "name": "Caribbean", + "parent": 254, + "level": 2, + "lft": 129, + "tree_id": 90, + "bbox_x0": -85.260178, + "bbox_x1": -59.286541, + "bbox_y0": 10.18548, + "bbox_y1": 27.454559 + } + }, + { + "pk": 256, + "model": "base.region", + "fields": { + "rght": 515, + "code": "PAC", + "name": "Pacific", + "parent": null, + "level": 1, + "lft": 464, + "tree_id": 90, + "bbox_x0": 112.921112, + "bbox_x1": -108.87291, + "bbox_y0": -54.640301, + "bbox_y1": 20.41712 + } + }, + { + "pk": 257, + "model": "base.region", + "fields": { + "rght": 12, + "code": "CFR", + "name": "Central Africa", + "parent": 10, + "level": 2, + "lft": 3, + "tree_id": 90, + "bbox_x0": -25.35874, + "bbox_x1": 63.525379, + "bbox_y0": -46.900452, + "bbox_y1": 37.56712 + } + }, + { + "pk": 258, + "model": "base.region", + "fields": { + "rght": 276, + "code": "EAS", + "name": "East Asia", + "parent": 6, + "level": 2, + "lft": 261, + "tree_id": 90, + "bbox_x0": 19.6381, + "bbox_x1": 180, + "bbox_y0": -12.56111, + "bbox_y1": 82.50045 + } + }, + { + "pk": 259, + "model": "base.region", + "fields": { + "rght": 59, + "code": "SSD", + "name": "South Sudan", + "parent": 11, + "level": 3, + "lft": 58, + "tree_id": 90, + "bbox_x0": 24.15192, + "bbox_x1": 35.947689, + "bbox_y0": 3.48639, + "bbox_y1": 12.21558 + } + } +] diff --git a/geonode/base/fixtures/sample_tags.json b/geonode/base/fixtures/sample_tags.json new file mode 100644 index 0000000..25ab91c --- /dev/null +++ b/geonode/base/fixtures/sample_tags.json @@ -0,0 +1,53 @@ +[ + { + "pk": 1, + "model": "taggit.tag", + "fields": { + "name": "usa", + "slug": "usa" + } + }, + { + "pk": 2, + "model": "taggit.tag", + "fields": { + "name": "relief", + "slug": "relief" + } + }, + { + "pk": 3, + "model": "taggit.tag", + "fields": { + "name": "isthmus", + "slug": "isthmus" + } + }, + { + "pk": 1, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": 20, + "object_id": 1 + } + }, + { + "pk": 2, + "model": "taggit.taggeditem", + "fields": { + "tag": 2, + "content_type": 20, + "object_id": 1 + } + }, + { + "pk": 3, + "model": "taggit.taggeditem", + "fields": { + "tag": 3, + "content_type": 20, + "object_id": 1 + } + } +] diff --git a/geonode/base/fixtures/test_data.json b/geonode/base/fixtures/test_data.json new file mode 100644 index 0000000..8213060 --- /dev/null +++ b/geonode/base/fixtures/test_data.json @@ -0,0 +1,56 @@ +[ + { + "pk": 1, + "model": "auth.group", + "fields": { + "name": "group1", + "permissions": [] + } + }, + { + "pk": 2, + "model": "auth.group", + "fields": { + "name": "group2", + "permissions": [] + } + }, + { + "pk": 1, + "model": "people.Profile", + "fields": { + "username": "bobby", + "first_name": "", + "last_name": "", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2010-06-10 16:59:13", + "groups": [], + "user_permissions": [], + "password": "sha1$8d019$a84eea3f5093eed93bc68bf62fe400f14042ab06", + "email": "bobby@bob.com", + "date_joined": "2010-06-10 16:58:18" + } + }, + { + "pk": 11, + "model": "people.Profile", + "fields": { + "username": "robert", + "first_name": "", + "last_name": "", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2010-06-10 16:59:13", + "groups": [ + 2 + ], + "user_permissions": [], + "password": "sha1$8d019$a84eea3f5093eed93bc68bf62fe400f14042ab06", + "email": "robert@bob.com", + "date_joined": "2010-06-10 16:58:18" + } + } +] diff --git a/geonode/base/fixtures/test_sld.sld b/geonode/base/fixtures/test_sld.sld new file mode 100644 index 0000000..d3211e6 --- /dev/null +++ b/geonode/base/fixtures/test_sld.sld @@ -0,0 +1,29 @@ + + + + Simple Point + + SLD Cook Book: Simple Point + + + + + + circle + + #FF0000 + + + 6 + + + + + + + diff --git a/geonode/base/fixtures/test_thesaurus.json b/geonode/base/fixtures/test_thesaurus.json new file mode 100644 index 0000000..5b2cb1c --- /dev/null +++ b/geonode/base/fixtures/test_thesaurus.json @@ -0,0 +1,714 @@ +[ + { + "model": "base.thesaurus", + "pk": 1, + "fields": { + "identifier": "inspire-theme", + "title": "GEMET - INSPIRE themes, version 1.0", + "date": "2018-05-23T10:25:56", + "description": "GEMET - INSPIRE themes, version 1.0", + "slug": "", + "about": "http://inspire.ec.europa.eu/theme", + "card_min": 1, + "card_max": 1, + "facet": true, + "order": 1 + } + },{ + "model": "base.thesaurus", + "pk": 2, + "fields": { + "identifier": "optional-theme", + "title": "Optional thesaurus Fields", + "date": "2021-05-23T10:25:56", + "description": "Optional thesaurus Fields", + "slug": "", + "about": "http://optional/theme2", + "card_min": 0, + "card_max": -1, + "facet": true, + "order": 0 + } + },{ + "model": "base.thesaurus", + "pk": 3, + "fields": { + "identifier": "no-about-thesauro", + "title": "Thesauro without the about", + "date": "2021-05-23T10:25:56", + "description": "Thesauro without the about", + "slug": "", + "about": "", + "card_min": 0, + "card_max": -1, + "facet": true, + "order": 2 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 1, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/ad", + "alt_label": "ad", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 2, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/au", + "alt_label": "au", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 3, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/rs", + "alt_label": "rs", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 4, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/gg", + "alt_label": "gg", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 5, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/cp", + "alt_label": "cp", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 6, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/gn", + "alt_label": "gn", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 7, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/hy", + "alt_label": "hy", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 8, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/ps", + "alt_label": "ps", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 9, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/tn", + "alt_label": "tn", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 10, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/el", + "alt_label": "el", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 11, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/ge", + "alt_label": "ge", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 12, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/lc", + "alt_label": "lc", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 13, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/oi", + "alt_label": "oi", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 14, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/af", + "alt_label": "af", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 15, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/am", + "alt_label": "am", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 16, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/ac", + "alt_label": "ac", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 17, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/br", + "alt_label": "br", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 18, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/bu", + "alt_label": "bu", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 19, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/er", + "alt_label": "er", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 20, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/ef", + "alt_label": "ef", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 21, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/hb", + "alt_label": "hb", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 22, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/hh", + "alt_label": "hh", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 23, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/lu", + "alt_label": "lu", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 24, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/mr", + "alt_label": "mr", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 25, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/nz", + "alt_label": "nz", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 26, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/of", + "alt_label": "of", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 27, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/pd", + "alt_label": "pd", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 28, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/pf", + "alt_label": "pf", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 29, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/sr", + "alt_label": "sr", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 30, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/so", + "alt_label": "so", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 31, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/sd", + "alt_label": "sd", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 32, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/su", + "alt_label": "su", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 34, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/us", + "alt_label": "us", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 35, + "fields": { + "about": "http://inspire.ec.europa.eu/theme/mf", + "alt_label": "mf", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 36, + "fields": { + "about": "", + "alt_label": "foo_keyword", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 37, + "fields": { + "about": "", + "alt_label": "", + "thesaurus": 1 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 38, + "fields": { + "about": "", + "alt_label": "", + "thesaurus": 3 + } + }, + { + "model": "base.thesauruskeyword", + "pk": 39, + "fields": { + "about": "", + "alt_label": "bar_keyword", + "thesaurus": 3 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 1, + "fields": { + "lang": "en", + "label": "Addresses", + "keyword": 1 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 2, + "fields": { + "lang": "de", + "label": "Adressen", + "keyword": 1 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 3, + "fields": { + "lang": "en", + "label": "Administrative units", + "keyword": 2 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 4, + "fields": { + "lang": "en", + "label": "Coordinate reference systems", + "keyword": 3 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 5, + "fields": { + "lang": "en", + "label": "Geographical grid systems", + "keyword": 4 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 6, + "fields": { + "lang": "en", + "label": "Cadastral parcels", + "keyword": 5 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 7, + "fields": { + "lang": "en", + "label": "Geographical names", + "keyword": 6 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 8, + "fields": { + "lang": "en", + "label": "Hydrography", + "keyword": 7 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 9, + "fields": { + "lang": "en", + "label": "Protected sites", + "keyword": 8 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 10, + "fields": { + "lang": "en", + "label": "Transport networks", + "keyword": 9 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 11, + "fields": { + "lang": "en", + "label": "Elevation", + "keyword": 10 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 12, + "fields": { + "lang": "en", + "label": "Geology", + "keyword": 11 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 13, + "fields": { + "lang": "en", + "label": "Land cover", + "keyword": 12 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 14, + "fields": { + "lang": "en", + "label": "Orthoimagery", + "keyword": 13 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 15, + "fields": { + "lang": "en", + "label": "Agricultural and aquaculture facilities", + "keyword": 14 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 16, + "fields": { + "lang": "en", + "label": "Area management/restriction/regulation zones and reporting units", + "keyword": 15 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 17, + "fields": { + "lang": "en", + "label": "Atmospheric conditions", + "keyword": 16 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 18, + "fields": { + "lang": "en", + "label": "Bio-geographical regions", + "keyword": 17 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 19, + "fields": { + "lang": "en", + "label": "Buildings", + "keyword": 18 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 20, + "fields": { + "lang": "en", + "label": "Energy resources", + "keyword": 19 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 21, + "fields": { + "lang": "en", + "label": "Environmental monitoring facilities", + "keyword": 20 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 22, + "fields": { + "lang": "en", + "label": "Habitats and biotopes", + "keyword": 21 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 23, + "fields": { + "lang": "en", + "label": "Human health and safety", + "keyword": 22 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 24, + "fields": { + "lang": "en", + "label": "Land use", + "keyword": 23 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 25, + "fields": { + "lang": "en", + "label": "Mineral resources", + "keyword": 24 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 26, + "fields": { + "lang": "en", + "label": "Natural risk zones", + "keyword": 25 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 27, + "fields": { + "lang": "en", + "label": "Oceanographic geographical features", + "keyword": 26 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 28, + "fields": { + "lang": "en", + "label": "Population distribution - demography", + "keyword": 27 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 29, + "fields": { + "lang": "en", + "label": "Production and industrial facilities", + "keyword": 28 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 30, + "fields": { + "lang": "en", + "label": "Sea regions", + "keyword": 29 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 31, + "fields": { + "lang": "en", + "label": "Soil", + "keyword": 30 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 32, + "fields": { + "lang": "en", + "label": "Species distribution", + "keyword": 31 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 33, + "fields": { + "lang": "en", + "label": "Statistical units", + "keyword": 32 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 34, + "fields": { + "lang": "en", + "label": "Utility and governmental services", + "keyword": 34 + } + }, + { + "model": "base.thesauruskeywordlabel", + "pk": 35, + "fields": { + "lang": "en", + "label": "Meteorological geographical features", + "keyword": 35 + } + }, + { + "model": "base.thesauruslabel", + "pk": 1, + "fields": { + "lang": "it", + "label": "Tema GEMET - INSPIRE, versione 1.0", + "thesaurus": 1 + } + } +] \ No newline at end of file diff --git a/geonode/base/fixtures/test_xml.xml b/geonode/base/fixtures/test_xml.xml new file mode 100644 index 0000000..1b99034 --- /dev/null +++ b/geonode/base/fixtures/test_xml.xml @@ -0,0 +1,735 @@ + + + + + + 7cfbc42c-efa7-431c-8daa-1399dff4cd19 + + + eng + + + utf8 + + + dataset + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ad@m.in + + + + + + + + pointOfContact + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ad@m.in + + + + + + + + author + + + + + + + + 2021-04-09T09:00:46Z + + + ISO 19115:2003 - Geographic information - Metadata + + + ISO 19115:2003 + + + + + + + + 4326 + + + EPSG + + + 6.11 + + + + + + + + + + + + + geonode:test_dataset + + + + test_dataset + + + + + 2021-04-09T08:49:00Z + + + publication + + + + + + + + + mapDigital + + + + + real abstract + + + + + + completed + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ad@m.in + + + + + + + + originator + + + + + + + + + + /static/geonode/img/missing_thumb.png?v=d498e972 + + + Thumbnail for 'test_dataset' + + + image/png + + + + + + + + ESRI Shapefile + + + + 1.0 + + + + + + + + + features + + + + test_dataset + + + + theme + + + + + + + + + + + + + + + + no conditions to access and use + + + + + + + Test for ordering + + + + + 2020-10-30T16:58:34 + + + pubblicazione + + + + + + + + + + + + + + ad + + + + + + af + + + + + + + + + GEMET - INSPIRE themes, version 1.0 + + + + + 2008-06-01 + + + pubblicazione + + + + + + + + + + + + + + Global + + + place + + + + + + + + + + license + + + Not Specified: The original author did not specify a license. + + + + + + + + + + + + + + + + vector + + + eng + + + utf8 + + + + + + + + + + -180.0 + + + 180.0 + + + -90.0 + + + 90.0 + + + + + + + + No information provided + + + + + + + + 0 + + + + + + + + + + + + + + http://localhost:8000/datasets/geonode_data:geonode:test_dataset + + + WWW:LINK-1.0-http--link + + + Online link to the 'test_dataset' description on GeoNode + + + + + + + + http://localhost:8000/download/62 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.zip + + + test_dataset (Original Dataset Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WMS&request=GetMap&layers=geonode%3Atest_dataset&format=image%2Fjpeg&height=550&width=550&srs=EPSG%3A4326&bbox=-180.0000000%2C-90.0000000%2C180.0000000%2C90.0000000 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.jpg + + + test_dataset (JPEG Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WMS&request=GetMap&layers=geonode%3Atest_dataset&format=application%2Fpdf&height=550&width=550&srs=EPSG%3A4326&bbox=-180.0000000%2C-90.0000000%2C180.0000000%2C90.0000000 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.pdf + + + test_dataset (PDF Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WMS&request=GetMap&layers=geonode%3Atest_dataset&format=image%2Fpng&height=550&width=550&srs=EPSG%3A4326&bbox=-180.0000000%2C-90.0000000%2C180.0000000%2C90.0000000 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.png + + + test_dataset (PNG Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WFS&version=1.0.0&request=GetFeature&typename=geonode%3Atest_dataset&outputFormat=SHAPE-ZIP&srs=EPSG%3A4326&format_options=charset%3AUTF-8 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.zip + + + test_dataset (Zipped Shapefile Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WFS&version=1.0.0&request=GetFeature&typename=geonode%3Atest_dataset&outputFormat=gml2&srs=EPSG%3A4326 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.gml + + + test_dataset (GML 2.0 Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WFS&version=1.0.0&request=GetFeature&typename=geonode%3Atest_dataset&outputFormat=text%2Fxml%3B+subtype%3Dgml%2F3.1.1&srs=EPSG%3A4326 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.gml + + + test_dataset (GML 3.1.1 Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WFS&version=1.0.0&request=GetFeature&typename=geonode%3Atest_dataset&outputFormat=csv&srs=EPSG%3A4326 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.csv + + + test_dataset (CSV Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WFS&version=1.0.0&request=GetFeature&typename=geonode%3Atest_dataset&outputFormat=excel&srs=EPSG%3A4326 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.excel + + + test_dataset (Excel Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WFS&version=1.0.0&request=GetFeature&typename=geonode%3Atest_dataset&outputFormat=json&srs=EPSG%3A4326&srsName=EPSG%3A4326 + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.json + + + test_dataset (GeoJSON Format) + + + + + + + + http://localhost:8080/geoserver/ows?service=WMS&request=GetLegendGraphic&format=image/png&WIDTH=20&HEIGHT=20&LAYER=geonode:test_dataset&STYLE=point&legend_options=fontAntiAliasing:true;fontSize:12;forceLabels:on + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.png + + + test_dataset (Legend Format) + + + + + + + + /static/geonode/img/missing_thumb.png?v=409a3f9b + + + WWW:DOWNLOAD-1.0-http--download + + + test_dataset.png + + + test_dataset (Thumbnail Format) + + + + + + + + + http://localhost:8080/geoserver/ows + + + OGC:WMS + + + geonode:test_dataset + + + geonode Service - Provides Layer: test_dataset + + + + + + + + http://localhost:8080/geoserver/ows + + + OGC:WFS + + + geonode:test_dataset + + + geonode Service - Provides Layer: test_dataset + + + + + + + + + + + + + + dataset + + + + + + Created with GeoNode + + + + + + diff --git a/geonode/base/forms.py b/geonode/base/forms.py new file mode 100644 index 0000000..d91fc1d --- /dev/null +++ b/geonode/base/forms.py @@ -0,0 +1,752 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import re +import html +import json +import logging + +from django.db.models.query import QuerySet +from bootstrap3_datetime.widgets import DateTimePicker +from dal import autocomplete +import dal.forward +from django import forms +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group +from django.core import validators +from django.db.models import Prefetch, Q +from django.forms import models +from django.forms.fields import ChoiceField, MultipleChoiceField +from django.forms.utils import flatatt +from django.utils.encoding import force_str +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ +from modeltranslation.forms import TranslationModelForm +from taggit.forms import TagField +from tinymce.widgets import TinyMCE +from django.contrib.admin.utils import flatten +from django.utils.translation import get_language + +from geonode.base.enumerations import ALL_LANGUAGES +from geonode.base.models import ( + HierarchicalKeyword, + License, + LinkedResource, + Region, + ResourceBase, + Thesaurus, + ThesaurusKeyword, + ThesaurusKeywordLabel, + ThesaurusLabel, + TopicCategory, +) +from geonode.base.widgets import TaggitSelect2Custom, TaggitProfileSelect2Custom +from geonode.base.fields import MultiThesauriField +from geonode.documents.models import Document +from geonode.layers.models import Dataset +from geonode.base.utils import validate_extra_metadata, remove_country_from_languagecode +from geonode.people import Roles + +logger = logging.getLogger(__name__) + + +def get_tree_data(): + def rectree(parent, path): + children_list_of_tuples = list() + c = Region.objects.filter(parent=parent) + for child in c: + children_list_of_tuples.append(tuple((path + parent.name, tuple((child.id, child.name))))) + childrens = rectree(child, f"{parent.name}/") + if childrens: + children_list_of_tuples.extend(childrens) + + return children_list_of_tuples + + data = list() + try: + t = Region.objects.filter(Q(level=0) | Q(parent=None)) + for toplevel in t: + data.append(tuple((toplevel.id, toplevel.name))) + childrens = rectree(toplevel, "") + if childrens: + data.append(tuple((toplevel.name, childrens))) + except Exception: + pass + + return tuple(data) + + +class AdvancedModelChoiceIterator(models.ModelChoiceIterator): + def choice(self, obj): + return (self.field.prepare_value(obj), self.field.label_from_instance(obj), obj) + + +class CategoryChoiceField(forms.ModelChoiceField): + def _get_choices(self): + if hasattr(self, "_choices"): + return self._choices + + return AdvancedModelChoiceIterator(self) + + choices = property(_get_choices, ChoiceField._set_choices) + + def label_from_instance(self, obj): + return ( + '' + '' + '' + "
" + obj.gn_description + "" + ) + + +class RegionsMultipleChoiceField(forms.MultipleChoiceField): + def validate(self, value): + """ + Validates that the input is a list or tuple. + """ + if self.required and not value: + raise forms.ValidationError(self.error_messages["required"], code="required") + + +class RegionsSelect(forms.Select): + allow_multiple_selected = True + + def render(self, name, value, attrs=None, renderer=None): + if value is None: + value = [] + final_attrs = self.build_attrs(attrs) + final_attrs["name"] = name + output = [format_html('") + return mark_safe("\n".join(output)) + + def value_from_datadict(self, data, files, name): + try: + getter = data.getlist + except AttributeError: + getter = data.get + return getter(name) + + def render_option_value(self, selected_choices, option_value, option_label, data_section=None): + if option_value is None: + option_value = "" + option_value = force_str(option_value) + if option_value in selected_choices: + selected_html = mark_safe(" selected") + if not self.allow_multiple_selected: + # Only allow for a single selection. + selected_choices.remove(option_value) + else: + selected_html = "" + + label = force_str(option_label) + + if data_section is None: + data_section = "" + else: + data_section = force_str(data_section) + if "/" in data_section: + label = format_html("{} [{}]", label, data_section.rsplit("/", 1)[1]) + + return format_html( + '', data_section, option_value, selected_html, label + ) + + def render_options(self, selected_choices): + # Normalize to strings. + def _region_id_from_choice(choice): + if isinstance(choice, int) or (isinstance(choice, str) and choice.isdigit()): + return int(choice) + else: + return choice.id + + selected_choices = {force_str(_region_id_from_choice(v)) for v in selected_choices} + output = [] + + output.append(format_html('', "Global")) + for option_value, option_label in self.choices: + if not isinstance(option_label, (list, tuple)) and isinstance(option_label, str): + output.append(self.render_option_value(selected_choices, option_value, option_label)) + output.append("") + + for option_value, option_label in self.choices: + if isinstance(option_label, (list, tuple)) and not isinstance(option_label, str): + output.append(format_html('', force_str(option_value))) + for option in option_label: + if isinstance(option, (list, tuple)) and not isinstance(option, str): + if isinstance(option[1][0], (list, tuple)) and not isinstance(option[1][0], str): + for option_child in option[1][0]: + output.append( + self.render_option_value( + selected_choices, *option_child, data_section=force_str(option[1][0][0]) + ) + ) + else: + output.append( + self.render_option_value( + selected_choices, *option[1], data_section=force_str(option[0]) + ) + ) + else: + output.append( + self.render_option_value(selected_choices, *option, data_section=force_str(option_value)) + ) + output.append("") + + return "\n".join(output) + + +class CategoryForm(forms.Form): + category_choice_field = CategoryChoiceField( + required=False, + label=f"*{_('Category')}", + empty_label=None, + queryset=TopicCategory.objects.filter(is_choice=True).extra(order_by=["description"]), + ) + + def clean(self): + cleaned_data = self.data + ccf_data = cleaned_data.get("category_choice_field") + category_mandatory = getattr(settings, "TOPICCATEGORY_MANDATORY", False) + if category_mandatory and not ccf_data: + msg = _("Category is required.") + self._errors = self.error_class([msg]) + + # Always return the full collection of cleaned data. + return cleaned_data + + +class TKeywordForm(forms.ModelForm): + prefix = "tkeywords" + + class Meta: + model = Document + fields = ["tkeywords"] + + tkeywords = MultiThesauriField( + queryset=ThesaurusKeyword.objects.prefetch_related( + Prefetch("keyword", queryset=ThesaurusKeywordLabel.objects.filter(lang="en")) + ), + widget=autocomplete.ModelSelect2Multiple( + url="thesaurus_autocomplete", + ), + label=_("Keywords from Thesaurus"), + required=False, + help_text=_( + "List of keywords from Thesaurus", + ), + ) + + +THESAURUS_RESULT_LIST_SEPERATOR = ("", "-------") + + +class ThesaurusAvailableForm(forms.Form): + # seperator at beginning of thesaurus search result and between + # results found in local language and alt label + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + lang = get_language() + for item in Thesaurus.objects.all().order_by("order", "id"): + tname = self._get_thesauro_title_label(item, lang) + if item.card_max == 0: + continue + elif item.card_max == 1 and item.card_min == 0: + self.fields[f"{item.id}"] = self._define_choicefield(item, False, tname, lang) + elif item.card_max == 1 and item.card_min == 1: + self.fields[f"{item.id}"] = self._define_choicefield(item, True, tname, lang) + elif item.card_max == -1 and item.card_min == 0: + self.fields[f"{item.id}"] = self._define_multifield(item, False, tname, lang) + elif item.card_max == -1 and item.card_min == 1: + self.fields[f"{item.id}"] = self._define_multifield(item, True, tname, lang) + + def cleanx(self, x): + cleaned_values = [] + for key, value in x.items(): + if isinstance(value, QuerySet): + for y in value: + cleaned_values.append(y.id) + elif value: + cleaned_values.append(value) + return ThesaurusKeyword.objects.filter(id__in=flatten(cleaned_values)) + + def _define_multifield(self, item, required, tname, lang): + return MultipleChoiceField( + choices=self._get_thesauro_keyword_label(item, lang), + widget=autocomplete.Select2Multiple( + url=f"/base/thesaurus_available/?sysid={item.id}&lang={lang}", + attrs={"class": "treq" if required else ""}, + ), + label=f"{tname}", + required=False, + ) + + def _define_choicefield(self, item, required, tname, lang): + return models.ChoiceField( + label=f"{tname}", + required=False, + widget=forms.Select(attrs={"class": "treq" if required else ""}), + choices=self._get_thesauro_keyword_label(item, lang), + ) + + @staticmethod + def _get_thesauro_keyword_label(item, lang): + keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=item) + + # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") + if len(qs_keyword_ids) == 0: + lang = remove_country_from_languagecode(lang) + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") + + not_qs_ids = ( + ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids) + .order_by("keyword_id") + .distinct("keyword_id") + .values("keyword_id") + ) + + qs_local = list( + ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list( + "keyword_id", "label" + ) + ) + qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids).values_list("id", "alt_label")) + + return [THESAURUS_RESULT_LIST_SEPERATOR] + qs_local + [THESAURUS_RESULT_LIST_SEPERATOR] + qs_non_local + + @staticmethod + def _get_thesauro_title_label(item, lang): + lang = remove_country_from_languagecode(lang) + tname = ThesaurusLabel.objects.values_list("label", flat=True).filter(thesaurus=item).filter(lang=lang) + if not tname: + return Thesaurus.objects.get(id=item.id).title + return tname.first() + + +class ContactRoleMultipleChoiceField(forms.ModelMultipleChoiceField): + def clean(self, value) -> QuerySet: + try: + users = get_user_model().objects.filter(username__in=value) + except TypeError: + # value of not supported type ... + raise forms.ValidationError(_("Something went wrong in finding the profile(s) in a contact role form ...")) + return users + + +class LinkedResourceForm(forms.ModelForm): + linked_resources = forms.ModelMultipleChoiceField( + label=_("Related resources"), + required=False, + queryset=None, + widget=autocomplete.ModelSelect2Multiple(url="autocomplete_linked_resource"), + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # this is used to automatically validate the POSTed back values + self.fields["linked_resources"].queryset = ResourceBase.objects.exclude(pk=self.instance.id) + # these are the LinkedResource already linked to this resource + self.fields["linked_resources"].initial = LinkedResource.get_target_ids(self.instance).all() + # this is used by the autocomplete view to exclude current resource + self.fields["linked_resources"].widget.forward.append( + dal.forward.Const( + self.instance.id, + "exclude", + ) + ) + + class Meta: + model = ResourceBase + fields = ["linked_resources"] + + def save_linked_resources(self, links_field="linked_resources"): + # create and fetch desired links + target_ids = [] + for res in self.cleaned_data[links_field]: + LinkedResource.objects.get_or_create(source=self.instance, target=res, internal=False) + target_ids.append(res.pk) + + # delete remaining links + ( + LinkedResource.objects.filter(source_id=self.instance.id, internal=False) + .exclude(target_id__in=target_ids) + .delete() + ) + + +class ResourceBaseDateTimePicker(DateTimePicker): + def build_attrs(self, base_attrs=None, extra_attrs=None, **kwargs): + "Helper function for building an attribute dictionary." + if extra_attrs: + base_attrs.update(extra_attrs) + base_attrs.update(kwargs) + return super().build_attrs(base_attrs) + # return base_attrs + + +class ResourceBaseForm(TranslationModelForm, LinkedResourceForm): + """Base form for metadata, should be inherited by childres classes of ResourceBase""" + + abstract = forms.CharField(label=_("Abstract"), required=False, widget=TinyMCE()) + + purpose = forms.CharField(label=_("Purpose"), required=False, widget=TinyMCE()) + + constraints_other = forms.CharField(label=_("Other constraints"), required=False, widget=TinyMCE()) + + supplemental_information = forms.CharField(label=_("Supplemental information"), required=False, widget=TinyMCE()) + + ptype = forms.CharField(required=False) + sourcetype = forms.CharField(required=False) + + data_quality_statement = forms.CharField(label=_("Data quality statement"), required=False, widget=TinyMCE()) + + owner = forms.ModelChoiceField( + empty_label=_(Roles.OWNER.label), + label=_(Roles.OWNER.label), + required=True, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=autocomplete.ModelSelect2(url="autocomplete_profile"), + ) + + date = forms.DateTimeField( + label=_("Date"), + localize=True, + input_formats=["%Y-%m-%d %H:%M %p"], + widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}), + ) + + temporal_extent_start = forms.DateTimeField( + label=_("temporal extent start"), + required=False, + localize=True, + input_formats=["%Y-%m-%d %H:%M %p"], + widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}), + ) + + temporal_extent_end = forms.DateTimeField( + label=_("temporal extent end"), + required=False, + localize=True, + input_formats=["%Y-%m-%d %H:%M %p"], + widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}), + ) + + metadata_author = ContactRoleMultipleChoiceField( + label=_(Roles.METADATA_AUTHOR.label), + required=Roles.METADATA_AUTHOR.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + processor = ContactRoleMultipleChoiceField( + label=_(Roles.PROCESSOR.label), + required=Roles.PROCESSOR.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + publisher = ContactRoleMultipleChoiceField( + label=_(Roles.PUBLISHER.label), + required=Roles.PUBLISHER.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + custodian = ContactRoleMultipleChoiceField( + label=_(Roles.CUSTODIAN.label), + required=Roles.CUSTODIAN.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + poc = ContactRoleMultipleChoiceField( + label=_(Roles.POC.label), + required=Roles.POC.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + distributor = ContactRoleMultipleChoiceField( + label=_(Roles.DISTRIBUTOR.label), + required=Roles.DISTRIBUTOR.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + resource_user = ContactRoleMultipleChoiceField( + label=_(Roles.RESOURCE_USER.label), + required=Roles.RESOURCE_USER.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + resource_provider = ContactRoleMultipleChoiceField( + label=_(Roles.RESOURCE_PROVIDER.label), + required=Roles.RESOURCE_PROVIDER.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + originator = ContactRoleMultipleChoiceField( + label=_(Roles.ORIGINATOR.label), + required=Roles.ORIGINATOR.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + principal_investigator = ContactRoleMultipleChoiceField( + label=_(Roles.PRINCIPAL_INVESTIGATOR.label), + required=Roles.PRINCIPAL_INVESTIGATOR.is_required, + queryset=get_user_model().objects.exclude(username="AnonymousUser"), + widget=TaggitProfileSelect2Custom(url="autocomplete_profile"), + ) + + keywords = TagField( + label=_("Free-text Keywords"), + required=False, + help_text=_("A space or comma-separated list of keywords. Use the widget to select from Hierarchical tree."), + # widget=TreeWidget(url='autocomplete_hierachical_keyword'), #Needs updating to work with select2 + widget=TaggitSelect2Custom(url="autocomplete_hierachical_keyword"), + ) + + regions = RegionsMultipleChoiceField(label=_("Regions"), required=False, widget=RegionsSelect) + + regions.widget.attrs = {"size": 20} + + extra_metadata = forms.CharField( + required=False, + widget=forms.Textarea, + help_text=_( + 'Additional metadata, must be in format [\ + {"metadata_key": "metadata_value"},\ + {"metadata_key": "metadata_value"} \ + ]' + ), + ) + + def __init__(self, *args, **kwargs): + self.user = kwargs.pop("user", None) + super().__init__(*args, **kwargs) + self.fields["regions"].choices = get_tree_data() + self.can_change_perms = self.user and self.user.has_perm( + "change_resourcebase_permissions", self.instance.get_self_resource() + ) + if self.instance and self.instance.id and self.instance.metadata.exists(): + self.fields["extra_metadata"].initial = [x.metadata for x in self.instance.metadata.all()] + + for field in self.fields: + if field == "featured" and self.user and not self.user.is_superuser: + self.fields[field].disabled = True + help_text = self.fields[field].help_text + if help_text != "": + self.fields[field].widget.attrs.update( + { + "class": "has-popover", + "data-content": help_text, + "data-placement": "right", + "data-container": "body", + "data-html": "true", + } + ) + + if field in ["owner"] and not self.can_change_perms: + self.fields[field].disabled = True + + def disable_keywords_widget_for_non_superuser(self, user): + if settings.FREETEXT_KEYWORDS_READONLY and not user.is_superuser: + self["keywords"].field.disabled = True + + def clean_keywords(self): + keywords = self.cleaned_data["keywords"] + _unsescaped_kwds = [] + for k in keywords: + _k = ("%s" % re.sub(r"%([A-Z0-9]{2})", r"&#x\g<1>;", k.strip())).split(",") + if not isinstance(_k, str): + for _kk in [html.unescape(x.strip()) for x in _k]: + # Simulate JS Unescape + _kk = ( + _kk.replace("%u", r"\u") + .encode("unicode-escape") + .replace(b"\\\\u", b"\\u") + .decode("unicode-escape") + if "%u" in _kk + else _kk + ) + _hk = HierarchicalKeyword.objects.filter(name__iexact=f"{_kk.strip()}") + if _hk and len(_hk) > 0: + _unsescaped_kwds.append(str(_hk[0])) + else: + _unsescaped_kwds.append(str(_kk)) + else: + _hk = HierarchicalKeyword.objects.filter(name__iexact=_k.strip()) + if _hk and len(_hk) > 0: + _unsescaped_kwds.append(str(_hk[0])) + else: + _unsescaped_kwds.append(str(_k)) + return _unsescaped_kwds + + def clean_title(self): + title = self.cleaned_data.get("title", None) + if title: + title = title.replace(",", "_") + return title + + def clean_extra_metadata(self): + cleaned_data = self.cleaned_data.get("extra_metadata", []) + return json.dumps(validate_extra_metadata(cleaned_data, self.instance), indent=4) + + class Meta: + model = ResourceBase + + exclude = ( + "contacts", + "name", + "uuid", + "bbox_polygon", + "ll_bbox_polygon", + "srid", + "category", + "csw_typename", + "csw_schema", + "csw_mdsource", + "csw_type", + "csw_wkt_geometry", + "metadata_uploaded", + "metadata_xml", + "csw_anytext", + "popular_count", + "share_count", + "thumbnail", + "charset", + "rating", + "detail_url", + "tkeywords", + "users_geolimits", + "groups_geolimits", + "dirty_state", + "state", + "blob", + "files", + "was_approved", + "was_published", + ) + + +class ValuesListField(forms.Field): + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return [] + + value = [item.strip() for item in value.split(",") if item.strip()] + + return value + + def clean(self, value): + value = self.to_python(value) + self.validate(value) + self.run_validators(value) + return value + + +class BatchEditForm(forms.Form): + LANGUAGES = (("", "--------"),) + ALL_LANGUAGES + group = forms.ModelChoiceField(label=_("Group"), queryset=Group.objects.all(), required=False) + owner = forms.ModelChoiceField(label=_("Owner"), queryset=get_user_model().objects.all(), required=False) + category = forms.ModelChoiceField(label=_("Category"), queryset=TopicCategory.objects.all(), required=False) + license = forms.ModelChoiceField(label=_("License"), queryset=License.objects.all(), required=False) + regions = forms.ModelChoiceField(label=_("Regions"), queryset=Region.objects.all(), required=False) + date = forms.DateTimeField(label=_("Date"), required=False) + language = forms.ChoiceField( + label=_("Language"), + required=False, + choices=LANGUAGES, + ) + keywords = forms.CharField(required=False) + ids = forms.CharField(required=False, widget=forms.HiddenInput()) + + +def get_user_choices(): + try: + return [(x.pk, x.title) for x in Dataset.objects.all().order_by("id")] + except Exception: + return [] + + +class UserAndGroupPermissionsForm(forms.Form): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["layers"].label_from_instance = self.label_from_instance + + layers = MultipleChoiceField( + choices=get_user_choices, + widget=autocomplete.Select2Multiple(url="datasets_autocomplete"), + label="Datasets", + required=False, + ) + + permission_type = forms.ChoiceField( + required=True, + widget=forms.RadioSelect, + choices=( + ("view", "View"), + ("download", "Download"), + ("edit", "Edit"), + ), + ) + mode = forms.ChoiceField( + required=True, + widget=forms.RadioSelect, + choices=( + ("set", "Set"), + ("unset", "Unset"), + ), + ) + ids = forms.CharField(required=False, widget=forms.HiddenInput()) + + @staticmethod + def label_from_instance(obj): + return obj.title + + +class OwnerRightsRequestForm(forms.Form): + resource = forms.ModelChoiceField( + label=_("Resource"), queryset=ResourceBase.objects.all(), widget=forms.HiddenInput() + ) + reason = forms.CharField( + label=_("Reason"), widget=forms.Textarea, help_text=_("Short reasoning behind the request"), required=True + ) + + class Meta: + fields = ["reason", "resource"] + + +class ThesaurusImportForm(forms.Form): + rdf_file = forms.FileField() diff --git a/geonode/base/management/__init__.py b/geonode/base/management/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/base/management/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/base/management/command_utils.py b/geonode/base/management/command_utils.py new file mode 100644 index 0000000..e9bc3e5 --- /dev/null +++ b/geonode/base/management/command_utils.py @@ -0,0 +1,24 @@ +import logging + +DEFAULT_COMMAND_LOGGER_NAME = "geonode.commands" + + +def setup_logger(logger_name=DEFAULT_COMMAND_LOGGER_NAME, level=logging.INFO, **kwargs): + logger = logging.getLogger(logger_name) + + # remove previous handlers + for old_handler in logger.handlers: + logger.removeHandler(old_handler) + + format = "%(levelname)-7s %(asctime)s %(message)s" + + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(fmt=format)) + handler.setLevel(level) + + logger.addHandler(handler) + + logger.setLevel(level) + logger.propagate = False + + return logger diff --git a/geonode/base/management/commands/__init__.py b/geonode/base/management/commands/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/base/management/commands/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/base/management/commands/delete_resources.py b/geonode/base/management/commands/delete_resources.py new file mode 100644 index 0000000..42e08df --- /dev/null +++ b/geonode/base/management/commands/delete_resources.py @@ -0,0 +1,160 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import json + +# import needed to resolve model filters: +from django.db import transaction, IntegrityError +from django.core.management.base import BaseCommand, CommandError + +from geonode.maps.models import Map +from geonode.layers.models import Dataset +from geonode.documents.models import Document + + +class Command(BaseCommand): + + help = 'Delete resources meeting a certain condition' + + def add_arguments(self, parser): + + parser.add_argument( + '-c', + '--config', + dest='config_path', + help='Configuration file path. Default is: delete_resources.json') + + parser.add_argument( + '-l', + '--dataset_filters', + dest='dataset_filters', + nargs='+', + required=False, + ) + + parser.add_argument( + '-m', + '--map_filters', + dest='map_filters', + nargs='+', + required=False, + ) + + parser.add_argument( + '-d', + '--document_filters', + dest='document_filters', + nargs='+', + required=False, + ) + + def handle(self, **options): + dataset_filters = options.get('dataset_filters') + map_filters = options.get('map_filters') + document_filters = options.get('document_filters') + + config_path = options.get('config_path') + + # check argument set + if all(config is None for config in {dataset_filters, map_filters, document_filters, config_path}): + raise CommandError( + "No configuration provided. Please specify any of the following arguments: '-l', '-m', '-d', '-c'.") + + if any([dataset_filters, map_filters, document_filters]) and config_path: + raise CommandError( + "Too many configuration options provided. Please use either '-c' or '-l', '-m', '-d' arguments.") + + # check config_file, if it exists + if config_path: + + if not os.path.exists(config_path): + raise CommandError(f'Specified configuration file does not exist: "{config_path}"') + + if os.path.getsize(config_path) == 0: + raise CommandError(f'Specified configuration file is empty: "{config_path}"') + + with open(config_path) as file: + try: + config = json.load(file) + except json.decoder.JSONDecodeError as e: + raise CommandError(f'Parsing configuration file failed with an exception: {e}') + + if ( + # if config is an empty JSON object + not config + # or 'filters' is not set in config OR it is not an JSON object + or not isinstance(config.get('filters'), dict) + # or all filters are empty + or not any( + [ + config.get('filters').get('dataset'), + config.get('filters').get('map'), + config.get('filters').get('document') + ]) + ): + print('Nothing to be done... exiting delete_resources command.') + return + + # override filters variables with configuration file data + dataset_filters = config.get('filters').get('dataset') + map_filters = config.get('filters').get('map') + document_filters = config.get('filters').get('document') + + # remove layers + if dataset_filters: + + if '*' in dataset_filters: + layers_to_delete = Dataset.objects.all() + else: + layers_q_expressions = [eval(expr) for expr in dataset_filters] + layers_to_delete = Dataset.objects.filter(*layers_q_expressions) + + for layer in layers_to_delete: + print(f'Deleting layer "{layer.name}" with ID: {layer.id}') + try: + with transaction.atomic(): + layer.delete() + except IntegrityError: + raise + + if map_filters: + + if '*' in map_filters: + maps_to_delete = Map.objects.all() + else: + maps_q_expressions = [eval(expr) for expr in map_filters] + maps_to_delete = Map.objects.filter(*maps_q_expressions) + + for map in maps_to_delete: + print(f'Deleting map "{map.title}" with ID: {map.id}') + map.maplayers.all().delete() + map.delete() + + if document_filters: + + if '*' in document_filters: + documents_to_delete = Document.objects.all() + else: + documents_q_expressions = [eval(expr) for expr in document_filters] + documents_to_delete = Document.objects.filter(*documents_q_expressions) + + for document in documents_to_delete: + print(f'Deleting document "{document.title}" with ID: {document.id}') + document.delete() diff --git a/geonode/base/management/commands/dump_thesaurus.py b/geonode/base/management/commands/dump_thesaurus.py new file mode 100644 index 0000000..9b39ae4 --- /dev/null +++ b/geonode/base/management/commands/dump_thesaurus.py @@ -0,0 +1,125 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# Copyright (C) 2022 King's College London +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from rdflib import Graph, URIRef, Literal +from rdflib.namespace import DC, DCTERMS, RDF, SKOS + +from geonode.base.models import Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel, ThesaurusLabel + + +class Command(BaseCommand): + + help = 'Dump a thesaurus in RDF format' + formats = sorted(['ttl', 'xml', 'pretty-xml', 'json-ld', 'nt', 'n3', 'trig']) + + def add_arguments(self, parser): + + # Named (optional) arguments + parser.add_argument( + '-n', + '--name', + dest='name', + help='Dump the thesaurus with the given name') + + parser.add_argument( + '-f', + '--format', + dest='format', + default='pretty-xml', + help=f'Format string supported by rdflib, e.g.: pretty-xml (default), {", ".join(self.formats)}' + ) + + parser.add_argument( + '--default-lang', + dest='lang', + default=getattr(settings, 'THESAURUS_DEFAULT_LANG', None), + help='Default language code for untagged string literals' + ) + + # Named (optional) arguments + parser.add_argument( + '-l', + '--list', + action="store_true", + dest='list', + default=False, + help='List available thesauri') + + def handle(self, **options): + + name = options.get('name') + list = options.get('list') + + if not name and not list: + raise CommandError("Missing identifier name for the thesaurus (--name)") + + if options.get('format') not in self.formats: + raise CommandError(f"Invalid output format: supported formats are {', '.join(self.formats)}") + + if list: + self.list_thesauri() + return + + self.dump_thesaurus(name, options.get('format'), options.get('lang')) + + def list_thesauri(self): + self.stderr.write(self.style.SUCCESS('LISTING THESAURI')) + + thesaurus_entries = Thesaurus.objects.values_list('identifier', flat=True) + if len(thesaurus_entries) == 0: + self.stderr.write(self.style.WARNING('NO ENTRIES FOUND ...')) + return + max_id_len = len(max(thesaurus_entries, key=len)) + + for t in Thesaurus.objects.order_by('order').all(): + if t.card_max == 0: + card = 'DISABLED' + else: + # DISABLED + # [0..n] + card = f'[{t.card_min}..{t.card_max if t.card_max!=-1 else "N"}] ' + self.stdout.write(f'id:{t.id:2} sort:{t.order:3} {card} name={t.identifier.ljust(max_id_len)} title="{t.title}" URI:{t.about}\n') + + def dump_thesaurus(self, name: str, fmt: str, default_lang: str): + + g = Graph() + thesaurus = Thesaurus.objects.filter(identifier=name).get() + scheme = URIRef(thesaurus.about) + g.add((scheme, RDF.type, SKOS.ConceptScheme)) + g.add((scheme, DC.title, Literal(thesaurus.title, lang=default_lang))) + g.add((scheme, DC.description, Literal(thesaurus.description, lang=default_lang))) + g.add((scheme, DCTERMS.issued, Literal(thesaurus.date))) + + for title_label in ThesaurusLabel.objects.filter(thesaurus=thesaurus).all(): + g.add((scheme, DC.title, Literal(title_label.label, lang=title_label.lang))) + + # Concepts + for keyword in ThesaurusKeyword.objects.filter(thesaurus=thesaurus).all(): + concept = URIRef(keyword.about) + g.add((concept, RDF.type, SKOS.Concept)) + g.add((concept, SKOS.inScheme, scheme)) + if keyword.alt_label: + g.add((concept, SKOS.altLabel, Literal(keyword.alt_label, lang=default_lang))) + for label in ThesaurusKeywordLabel.objects.filter(keyword=keyword).all(): + g.add((concept, SKOS.prefLabel, Literal(label.label, lang=label.lang))) + + self.stdout.write(g.serialize(format=fmt)) diff --git a/geonode/base/management/commands/fixoauthuri.py b/geonode/base/management/commands/fixoauthuri.py new file mode 100644 index 0000000..92944ed --- /dev/null +++ b/geonode/base/management/commands/fixoauthuri.py @@ -0,0 +1,84 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand +from oauth2_provider.models import Application +from oauth2_provider.generators import generate_client_id, generate_client_secret + +from geonode import geoserver # noqa +from geonode.utils import check_ogc_backend + + +class Command(BaseCommand): + """Creates or updates the oauth2 Application + """ + can_import_settings = True + + def add_arguments(self, parser): + + # Named (optional) arguments + parser.add_argument( + '-f', + '--force', + action='store_true', + dest='force_exec', + default=False, + help='Forces the regeneration of OAUth keys.') + + parser.add_argument( + '--target-address', + dest='target_address', + help='Target Address (the one to be changed e.g. http://my-public.geonode.org)') + + def handle(self, *args, **options): + + force_exec = options.get('force_exec') + target_address = options.get('target_address') + + client_id = None + client_secret = None + + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + from geonode.geoserver.helpers import ogc_server_settings + redirect_uris = f'{ogc_server_settings.LOCATION}\n{ogc_server_settings.public_url}\n{target_address}/geoserver/' # noqa + if Application.objects.filter(name='GeoServer').exists(): + Application.objects.filter(name='GeoServer').update(redirect_uris=redirect_uris) + if force_exec: + Application.objects.filter(name='GeoServer').update( + client_id=generate_client_id(), + client_secret=generate_client_secret() + ) + app = Application.objects.filter(name='GeoServer')[0] + client_id = app.client_id + client_secret = app.client_secret + else: + client_id = generate_client_id() + client_secret = generate_client_secret() + Application.objects.create( + skip_authorization=True, + redirect_uris=redirect_uris, + name='GeoServer', + authorization_grant_type='authorization-code', + client_type='confidential', + client_id=client_id, + client_secret=client_secret, + user=get_user_model().objects.filter(is_superuser=True)[0] + ) + return f'{client_id},{client_secret}' diff --git a/geonode/base/management/commands/fixsitename.py b/geonode/base/management/commands/fixsitename.py new file mode 100644 index 0000000..3a1caef --- /dev/null +++ b/geonode/base/management/commands/fixsitename.py @@ -0,0 +1,41 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand +from django.contrib.sites.models import Site +from urllib.parse import urlsplit + + +class Command(BaseCommand): + """Overrides the default Site object with information from + SITENAME and SITEURL + """ + can_import_settings = True + + def handle(self, *args, **options): + from django.conf import settings + name = getattr(settings, 'SITENAME', 'GeoNode') + site_url = getattr(settings, 'SITEURL') + url = site_url.rstrip('/') if site_url.startswith('http') else site_url + parsed = urlsplit(url) + + site = Site.objects.get_current() + site.name = name + site.domain = parsed.netloc + site.save() diff --git a/geonode/base/management/commands/helpers.py b/geonode/base/management/commands/helpers.py new file mode 100644 index 0000000..226fe55 --- /dev/null +++ b/geonode/base/management/commands/helpers.py @@ -0,0 +1,62 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "lib")) + + +def confirm(prompt=None, resp=False): + """prompts for yes or no response from the user. Returns True for yes and + False for no. + + 'resp' should be set to the default value assumed by the caller when + user simply types ENTER. + + >>> confirm(prompt='Create Directory?', resp=True) + Create Directory? [y]|n: + True + >>> confirm(prompt='Create Directory?', resp=False) + Create Directory? [n]|y: + False + >>> confirm(prompt='Create Directory?', resp=False) + Create Directory? [n]|y: y + True + """ + + if prompt is None: + prompt = 'Confirm' + + if resp: + prompt = f'{prompt} [y]|n: ' + else: + prompt = f'{prompt} [n]|y: ' + + while True: + ans = input(prompt) + if not ans: + return resp + if ans not in {'y', 'Y', 'n', 'N'}: + print('please enter y or n.') + continue + if ans == 'y' or ans == 'Y': + return True + if ans == 'n' or ans == 'N': + return False diff --git a/geonode/base/management/commands/lib/gn20_to_24.py b/geonode/base/management/commands/lib/gn20_to_24.py new file mode 100644 index 0000000..e43a5fc --- /dev/null +++ b/geonode/base/management/commands/lib/gn20_to_24.py @@ -0,0 +1,281 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import re +import datetime +import json +from django.utils import timezone + + +class DefaultMangler(json.JSONDecoder): + + def __init__(self, *args, **kwargs): + + self.basepk = kwargs.get('basepk', -1) + self.owner = kwargs.get('owner', 'admin') + self.datastore = kwargs.get('datastore', '') + self.siteurl = kwargs.get('siteurl', '') + + super().__init__(*args) + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + + return default_obj + + +class ResourceBaseMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + upload_sessions = [] + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + obj['fields']['featured'] = False + obj['fields']['rating'] = 0 + obj['fields']['popular_count'] = 0 + obj['fields']['share_count'] = 0 + obj['fields']['is_published'] = True + obj['fields']['thumbnail_url'] = '' + + if 'distribution_url' in obj['fields']: + if not obj['fields']['distribution_url'] is None and 'layers' in obj['fields']['distribution_url']: + + obj['fields']['polymorphic_ctype'] = ["layers", "layer"] + + try: + p = '(?Phttp.*://)?(?P[^:/ ]+).?(?P[0-9]*)(?P.*)' + m = re.search(p, obj['fields']['distribution_url']) + if 'http' in m.group('protocol'): + obj['fields']['detail_url'] = self.siteurl + m.group('details_url') + else: + obj['fields']['detail_url'] = self.siteurl + obj['fields']['distribution_url'] + except Exception: + obj['fields']['detail_url'] = obj['fields']['distribution_url'] + + else: + obj['fields']['polymorphic_ctype'] = ["maps", "map"] + + try: + obj['fields'].pop("distribution_description", None) + except Exception: + pass + + try: + obj['fields'].pop("distribution_url", None) + except Exception: + pass + + try: + obj['fields'].pop("thumbnail", None) + except Exception: + pass + + upload_sessions.append(self.add_upload_session(obj['pk'], obj['fields']['owner'])) + + default_obj.extend(upload_sessions) + + return default_obj + + def add_upload_session(self, pk, owner): + obj = dict() + + obj['pk'] = pk + obj['model'] = 'datasets.uploadsession' + + obj['fields'] = dict() + obj['fields']['user'] = owner + obj['fields']['traceback'] = None + obj['fields']['context'] = None + obj['fields']['error'] = None + obj['fields']['processed'] = True + obj['fields']['date'] = datetime.datetime.now(timezone.get_current_timezone()).strftime("%Y-%m-%dT%H:%M:%S") + + return obj + + +class LayerMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + # Retrieve the ResourceBase associated to this Dataset + from geonode.base.models import ResourceBase + + resource = ResourceBase.objects.get(pk=obj['pk']) + + obj['fields']['upload_session'] = obj['pk'] + obj['fields']['service'] = None + obj['fields']['charset'] = "UTF-8" + obj['fields']['title_en'] = resource.title + obj['fields']['data_quality_statement_en'] = "" + obj['fields']['regions'] = [] + obj['fields']['supplemental_information_en'] = "No information provided" + obj['fields']['abstract_en'] = "No abstract provided" + obj['fields']['purpose_en'] = "" + obj['fields']['constraints_other_en'] = "" + obj['fields']['default_style'] = None + + if self.datastore: + obj['fields']['store'] = self.datastore + else: + obj['fields']['store'] = obj['fields']['name'] + + try: + obj['fields'].pop("popular_count", None) + except Exception: + pass + + try: + obj['fields'].pop("share_count", None) + except Exception: + pass + + try: + obj['fields'].pop("title", None) + except Exception: + pass + + return default_obj + + +class LayerAttributesMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + obj['fields']['layer'] = obj['fields']['layer'] + self.basepk + + return default_obj + + +class MapMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + # Retrieve the ResourceBase associated to this Dataset + from geonode.base.models import ResourceBase + + resource = ResourceBase.objects.get(pk=obj['pk']) + + obj['fields']['urlsuffix'] = "" + obj['fields']['title_en'] = resource.title + obj['fields']['featuredurl'] = "" + obj['fields']['data_quality_statement_en'] = None + obj['fields']['supplemental_information_en'] = "No information provided" + obj['fields']['abstract_en'] = "" + obj['fields']['purpose_en'] = None + obj['fields']['constraints_other_en'] = None + + try: + obj['fields'].pop("popular_count", None) + except Exception: + pass + + try: + obj['fields'].pop("share_count", None) + except Exception: + pass + + try: + obj['fields'].pop("title", None) + except Exception: + pass + + return default_obj + + +class MapLayersMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + obj['fields']['map'] = obj['fields']['map'] + self.basepk + + return default_obj diff --git a/geonode/base/management/commands/lib/gn24_to_24.py b/geonode/base/management/commands/lib/gn24_to_24.py new file mode 100644 index 0000000..30794ea --- /dev/null +++ b/geonode/base/management/commands/lib/gn24_to_24.py @@ -0,0 +1,179 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import re +import datetime +import json +from django.utils import timezone + + +class DefaultMangler(json.JSONDecoder): + + def __init__(self, *args, **kwargs): + + self.basepk = kwargs.get('basepk', -1) + self.owner = kwargs.get('owner', 'admin') + self.datastore = kwargs.get('datastore', '') + self.siteurl = kwargs.get('siteurl', '') + + super().__init__(*args) + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + return default_obj + + +class ResourceBaseMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + upload_sessions = [] + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + obj['fields']['owner'] = [self.owner] + + if 'distribution_url' in obj['fields']: + if not obj['fields']['distribution_url'] is None and 'layers' in obj['fields']['distribution_url']: + try: + p = '(?Phttp.*://)?(?P[^:/ ]+).?(?P[0-9]*)(?P.*)' + m = re.search(p, obj['fields']['distribution_url']) + if 'http' in m.group('protocol'): + obj['fields']['detail_url'] = self.siteurl + m.group('details_url') + else: + obj['fields']['detail_url'] = self.siteurl + obj['fields']['distribution_url'] + except Exception: + obj['fields']['detail_url'] = obj['fields']['distribution_url'] + + upload_sessions.append(self.add_upload_session(obj['pk'], obj['fields']['owner'])) + + default_obj.extend(upload_sessions) + + return default_obj + + def add_upload_session(self, pk, owner): + obj = dict() + + obj['pk'] = pk + obj['model'] = 'datasets.uploadsession' + + obj['fields'] = dict() + obj['fields']['user'] = owner + obj['fields']['traceback'] = None + obj['fields']['context'] = None + obj['fields']['error'] = None + obj['fields']['processed'] = True + obj['fields']['date'] = datetime.datetime.now(timezone.get_current_timezone()).strftime("%Y-%m-%dT%H:%M:%S") + + return obj + + +class LayerMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + obj['fields']['upload_session'] = obj['pk'] + obj['fields']['service'] = None + + if self.datastore: + obj['fields']['store'] = self.datastore + else: + obj['fields']['store'] = obj['fields']['name'] + + return default_obj + + +class LayerAttributesMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + obj['fields']['layer'] = obj['fields']['layer'] + self.basepk + + return default_obj + + +class MapLayersMangler(DefaultMangler): + + def default(self, obj): + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + def decode(self, json_string): + """ + json_string is basicly string that you give to json.loads method + """ + default_obj = super().decode(json_string) + + # manipulate your object any way you want + # .... + for obj in default_obj: + obj['pk'] = obj['pk'] + self.basepk + + obj['fields']['map'] = obj['fields']['map'] + self.basepk + + return default_obj diff --git a/geonode/base/management/commands/load_thesaurus.py b/geonode/base/management/commands/load_thesaurus.py new file mode 100644 index 0000000..a67a512 --- /dev/null +++ b/geonode/base/management/commands/load_thesaurus.py @@ -0,0 +1,231 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# Copyright (C) 2022 King's College London +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from typing import List + +from django.conf import settings +from django.core.files.uploadedfile import UploadedFile +from django.core.management.base import BaseCommand, CommandError +from rdflib import Graph, Literal +from rdflib.namespace import RDF, RDFS, SKOS, DC, DCTERMS +from rdflib.util import guess_format + +from geonode.base.models import Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel, ThesaurusLabel + + +class Command(BaseCommand): + help = "Load a thesaurus in RDF format into DB" + + def add_arguments(self, parser): + # Named (optional) arguments + parser.add_argument( + "-d", + "--dry-run", + action="store_true", + dest="dryrun", + default=False, + help="Only parse and print the thesaurus file, without perform insertion in the DB.", + ) + + parser.add_argument("--name", dest="name", help="Identifier name for the thesaurus in this GeoNode instance.") + + parser.add_argument("--file", dest="file", help="Full path to a thesaurus in RDF format.") + + def handle(self, **options): + input_file = options.get("file") + name = options.get("name") + dryrun = options.get("dryrun") + + if not input_file: + raise CommandError("Missing thesaurus rdf file path (--file)") + + if not name: + raise CommandError("Missing identifier name for the thesaurus (--name)") + + if name.startswith("fake"): + self.create_fake_thesaurus(name) + else: + self.load_thesaurus(input_file, name, not dryrun) + + def load_thesaurus(self, input_file, name, store): + g = Graph() + + # if the input_file is an UploadedFile object rather than a file path the Graph.parse() + # method may not have enough info to correctly guess the type; in this case supply the + # name, which should include the extension, to guess_format manually... + rdf_format = None + if isinstance(input_file, UploadedFile): + self.stderr.write(self.style.WARNING(f"Guessing RDF format from {input_file.name}...")) + rdf_format = guess_format(input_file.name) + + g.parse(input_file, format=rdf_format) + + # An error will be thrown here there is more than one scheme in the file + scheme = g.value(None, RDF.type, SKOS.ConceptScheme, any=False) + if scheme is None: + raise CommandError("ConceptScheme not found in file") + + default_lang = getattr(settings, "THESAURUS_DEFAULT_LANG", None) + + available_titles = [t for t in g.objects(scheme, DC.title) if isinstance(t, Literal)] + thesaurus_title = value_for_language(available_titles, default_lang) + description = g.value(scheme, DC.description, None, default=thesaurus_title) + date_issued = g.value(scheme, DCTERMS.issued, None, default="") + + self.stderr.write( + self.style.SUCCESS(f'Thesaurus "{thesaurus_title}", desc: {description} issued at {date_issued}') + ) + + thesaurus = Thesaurus() + thesaurus.identifier = name + thesaurus.description = description + thesaurus.title = thesaurus_title + thesaurus.about = str(scheme) + thesaurus.date = date_issued + + if store: + thesaurus.save() + + for lang in available_titles: + if lang.language is not None: + thesaurus_label = ThesaurusLabel() + thesaurus_label.lang = lang.language + thesaurus_label.label = lang.value + thesaurus_label.thesaurus = thesaurus + + if store: + thesaurus_label.save() + + for concept in g.subjects(RDF.type, SKOS.Concept): + pref = preferredLabel(g, concept, default_lang)[0][1] + about = str(concept) + alt_label = g.value(concept, SKOS.altLabel, object=None, default=None) + if alt_label is not None: + alt_label = str(alt_label) + else: + available_labels = [t for t in g.objects(concept, SKOS.prefLabel) if isinstance(t, Literal)] + alt_label = value_for_language(available_labels, default_lang) + + self.stderr.write(self.style.SUCCESS(f"Concept {str(pref)}: {alt_label} ({about})")) + + tk = ThesaurusKeyword() + tk.thesaurus = thesaurus + tk.about = about + tk.alt_label = alt_label + + if store: + tk.save() + + for _, pref_label in preferredLabel(g, concept): + lang = pref_label.language + label = str(pref_label) + self.stderr.write(self.style.SUCCESS(f" Label {lang}: {label}")) + + tkl = ThesaurusKeywordLabel() + tkl.keyword = tk + tkl.lang = lang + tkl.label = label + + if store: + tkl.save() + + def create_fake_thesaurus(self, name): + thesaurus = Thesaurus() + thesaurus.identifier = name + + thesaurus.title = f"Title: {name}" + thesaurus.description = "SAMPLE FAKE THESAURUS USED FOR TESTING" + thesaurus.date = "2016-10-01" + + thesaurus.save() + + for keyword in ["aaa", "bbb", "ccc"]: + tk = ThesaurusKeyword() + tk.thesaurus = thesaurus + tk.about = f"{keyword}_about" + tk.alt_label = f"{keyword}_alt" + tk.save() + + for _l in ["it", "en", "es"]: + tkl = ThesaurusKeywordLabel() + tkl.keyword = tk + tkl.lang = _l + tkl.label = f"{keyword}_l_{_l}_t_{name}" + tkl.save() + + +def value_for_language(available: List[Literal], default_lang: str) -> str: + sorted_lang = sorted(available, key=lambda literal: "" if literal.language is None else literal.language) + for item in sorted_lang: + if item.language is None: + return str(item) + elif item.language.split("-")[0] == default_lang: + return str(item) + return str(available[0]) + + +def preferredLabel( + g, + subject, + lang=None, + default=None, + label_properties=(SKOS.prefLabel, RDFS.label), +): + """ + Find the preferred label for subject. + + By default prefers skos:prefLabels over rdfs:labels. In case at least + one prefLabel is found returns those, else returns labels. In case a + language string (e.g., "en", "de" or even "" for no lang-tagged + literals) is given, only such labels will be considered. + + Return a list of (labelProp, label) pairs, where labelProp is either + skos:prefLabel or rdfs:label. + + Copied from rdflib 6.1.1 + """ + + if default is None: + default = [] + + # setup the language filtering + if lang is not None: + if lang == "": # we only want not language-tagged literals + + def langfilter(l_): + return l_.language is None + + else: + + def langfilter(l_): + return l_.language == lang + + else: # we don't care about language tags + + def langfilter(l_): + return True + + for labelProp in label_properties: + labels = list(filter(langfilter, g.objects(subject, labelProp))) + if len(labels) == 0: + continue + else: + return [(labelProp, l_) for l_ in labels] + return default diff --git a/geonode/base/management/commands/migrate_baseurl.py b/geonode/base/management/commands/migrate_baseurl.py new file mode 100644 index 0000000..1d3a4b2 --- /dev/null +++ b/geonode/base/management/commands/migrate_baseurl.py @@ -0,0 +1,156 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import json + +from . import helpers + +from django.db.models import Func, F, Value +from django.contrib.sites.models import Site +from django.core.management.base import BaseCommand, CommandError + +from oauth2_provider.models import Application + +from geonode import geoserver +from geonode.base.models import Link +from geonode.utils import check_ogc_backend +from geonode.base.models import ResourceBase +from geonode.maps.models import Map, MapLayer +from geonode.layers.models import Dataset, Style +import logging + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + help = 'Migrate GeoNode VM Base URL' + + def add_arguments(self, parser): + + # Named (optional) arguments + parser.add_argument( + '-f', + '--force', + action='store_true', + dest='force_exec', + default=False, + help='Forces the execution without asking for confirmation.') + + parser.add_argument( + '--source-address', + dest='source_address', + help='Source Address (the one currently on DB e.g. http://192.168.1.23)') + + parser.add_argument( + '--target-address', + dest='target_address', + help='Target Address (the one to be changed e.g. http://my-public.geonode.org)') + + def handle(self, **options): + force_exec = options.get('force_exec') + source_address = options.get('source_address') + target_address = options.get('target_address') + + if not source_address or len(source_address) == 0: + raise CommandError("Source Address '--source-address' is mandatory") + + if not target_address or len(target_address) == 0: + raise CommandError("Target Address '--target-address' is mandatory") + + print(f"This will change all Maps, Layers, \ +Styles and Links Base URLs from [{source_address}] to [{target_address}].") + print("The operation may take some time, depending on the amount of Dataset on GeoNode.") + message = 'You want to proceed?' + + if force_exec or helpers.confirm(prompt=message, resp=False): + try: + _cnt = Map.objects.filter(thumbnail_url__icontains=source_address).update( + thumbnail_url=Func( + F('thumbnail_url'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} Maps Thumbnail URLs") + logger.info(f"Updated {_cnt} Maps Thumbnail URLs") + + _cnt = MapLayer.objects.filter(ows_url__icontains=source_address).update( + ows_url=Func( + F('ows_url'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} MapLayers OWS URLs") + logger.info(f"Updated {_cnt} MapLayers OWS URLs") + + _cnt = Dataset.objects.filter(ows_url__icontains=source_address).update( + ows_url=Func( + F('ows_url'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} Datasets OWS URLs") + logger.info(f"Updated {_cnt} Datasets OWS URLs") + _cnt = Dataset.objects.filter(thumbnail_url__icontains=source_address).update( + thumbnail_url=Func( + F('thumbnail_url'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} Datasets Thumbnail URLs") + logger.info(f"Updated {_cnt} Datasets Thumbnail URLs") + + _cnt = Style.objects.filter(sld_url__icontains=source_address).update( + sld_url=Func( + F('sld_url'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} Styles URLs") + logger.info(f"Updated {_cnt} Styles URLs") + + _cnt = Link.objects.filter(url__icontains=source_address).update( + url=Func( + F('url'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} Links URLs") + logger.info(f"Updated {_cnt} Links URLs") + + _cnt = ResourceBase.objects.filter(thumbnail_url__icontains=source_address).update( + thumbnail_url=Func( + F('thumbnail_url'), Value(source_address), Value(target_address), function='replace')) + _cnt += ResourceBase.objects.filter(csw_anytext__icontains=source_address).update( + csw_anytext=Func( + F('csw_anytext'), Value(source_address), Value(target_address), function='replace')) + _cnt += ResourceBase.objects.filter(metadata_xml__icontains=source_address).update( + metadata_xml=Func( + F('metadata_xml'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} ResourceBases") + logger.info(f"Updated {_cnt} ResourceBases") + # update blob context + _cnt = 0 + for resource in ResourceBase.objects.filter(blob__icontains=source_address): + current_blob = json.dumps(resource.blob) + ResourceBase.objects.filter(id=resource.id).update(blob=json.loads(current_blob.replace(source_address, target_address))) + _cnt += 1 + print(f"Updated {_cnt} 'blob' contexts.") + logger.info(f"Updated {_cnt} 'blob' contexts.") + + site = Site.objects.get_current() + if site: + site.name = site.name.replace(source_address, target_address) + site.domain = site.domain.replace(source_address, target_address) + site.save() + print("Updated 1 Site") + logger.info("Updated 1 Site") + + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + if Application.objects.filter(name='GeoServer').exists(): + _cnt = Application.objects.filter(name='GeoServer').update( + redirect_uris=Func( + F('redirect_uris'), Value(source_address), Value(target_address), function='replace')) + print(f"Updated {_cnt} OAUth2 Redirect URIs") + logger.info(f"Updated {_cnt} OAUth2 Redirect URIs") + finally: + print("...done!") + logger.info("...done!") diff --git a/geonode/base/management/commands/migrate_notifications.py b/geonode/base/management/commands/migrate_notifications.py new file mode 100644 index 0000000..1bbb6ef --- /dev/null +++ b/geonode/base/management/commands/migrate_notifications.py @@ -0,0 +1,58 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.core.management.base import BaseCommand +from django.db import connection, IntegrityError +from django.db.utils import ProgrammingError + +from pinax.notifications.models import NoticeSetting + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Migrate geonode-notifications settings to pinax-notifications + """ + + def has_duplicate(self, ns): + args = dict(zip('medium send notice_type_id user_id'.split(' '), ns)) + return NoticeSetting.objects.filter(**args).exists() + + def handle(self, **options): + c = connection.cursor() + # check if old notifications exist + try: + c.execute('select medium, send, notice_type_id, user_id from notification_noticesetting;') + except ProgrammingError as err: + log.error(f"No table for notification app, exiting: {err}") + # no source of data, bye! + return + + for ns in c.fetchall(): + try: + if self.has_duplicate(ns): + log.debug('settings for %s have duplicate', ns) + continue + c.execute("""insert into pinax_notifications_noticesetting + (medium, send, notice_type_id, user_id) + values (%s, %s, %s, %s)""", ns) + except IntegrityError as err: + log.error('Cannot insert notifications for %s: %s', ns, err, exc_info=err) diff --git a/geonode/base/management/commands/set_all_datasets_alternate.py b/geonode/base/management/commands/set_all_datasets_alternate.py new file mode 100644 index 0000000..0a1ab34 --- /dev/null +++ b/geonode/base/management/commands/set_all_datasets_alternate.py @@ -0,0 +1,78 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand +from geonode.layers.models import Dataset +import logging + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Resets Permissions to Public for All Layers' + + def add_arguments(self, parser): + parser.add_argument( + '-i', + '--ignore-errors', + action='store_true', + dest='ignore_errors', + default=False, + help='Stop after any errors are encountered.' + ) + parser.add_argument( + '-f', + '--filter', + dest="filter", + default=None, + help="Only update data the layers that match the given filter"), + parser.add_argument( + '-u', + '--username', + dest="username", + default=None, + help="Only update data owned by the specified username") + + def handle(self, *args, **options): + ignore_errors = options.get('ignore_errors') + filter = options.get('filter') + if not options.get('username'): + username = None + else: + username = options.get('username') + + all_datasets = Dataset.objects.all().order_by('name') + + if filter: + all_datasets = all_datasets.filter(name__icontains=filter) + if username: + all_datasets = all_datasets.filter(owner__username=username) + + for index, layer in enumerate(all_datasets): + logger.info(f"[{(index + 1)} / {len(all_datasets)}] Checking 'alternate' of Dataset [{layer.name}] ...") + try: + if not layer.alternate: + layer.alternate = layer.typename + layer.save() + except Exception as e: + if ignore_errors: + logger.error(f"[ERROR] Dataset [{layer.name}] couldn't be updated") + else: + raise e diff --git a/geonode/base/management/commands/set_all_datasets_metadata.py b/geonode/base/management/commands/set_all_datasets_metadata.py new file mode 100644 index 0000000..995a62b --- /dev/null +++ b/geonode/base/management/commands/set_all_datasets_metadata.py @@ -0,0 +1,177 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand + +from geonode.layers.models import Dataset +from geonode import geoserver # noqa +from geonode.catalogue.models import catalogue_post_save +import logging +from django.conf import settings + +from geonode.utils import ( + check_ogc_backend, + set_resource_default_links +) +from geonode.base.utils import ( + delete_orphaned_thumbs, + remove_duplicate_links +) +import ast + +if check_ogc_backend(geoserver.BACKEND_PACKAGE): + from geonode.geoserver.helpers import set_attributes_from_geoserver as set_attributes + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Resets Metadata Attributes and Schema to All Layers' + + def add_arguments(self, parser): + parser.add_argument( + '-i', + '--ignore-errors', + action='store_true', + dest='ignore_errors', + default=False, + help='Stop after any errors are encountered.' + ) + + parser.add_argument( + '-d', + '--remove-duplicates', + action='store_true', + dest='remove_duplicates', + default=False, + help='Remove duplicates first.' + ) + + parser.add_argument( + '-p', + '--prune', + action='store_true', + dest='prune', + default=False, + help='Prune Old Links.' + ) + + parser.add_argument( + '-t', + '--delete-orphaned-thumbs', + action='store_true', + dest='delete_orphaned_thumbnails', + default=False, + help='Delete Orphaned Thumbnails.' + ) + + parser.add_argument( + '-f', + '--filter', + dest="filter", + default=None, + help="Only update data the layers that match the given filter"), + + parser.add_argument( + '-u', + '--username', + dest="username", + default=None, + help="Only update data owned by the specified username") + + parser.add_argument( + '-uuid', + '--set-uuid', + dest='set_uuid', + default="False", + help='if set as True, will refresh the UUID based on the UUID_HANDLER if configured (Default False)' + ) + + parser.add_argument( + '-attr', + '--set-attr', + dest='set_attrib', + default="True", + help='If set will refresh the attributes of the resource taken from Geoserver. (Default True)' + ) + + parser.add_argument( + '-l', + '--set-links', + dest='set_links', + default="True", + help='If set will refresh the links of the resource. (Default True)' + ) + + def handle(self, *args, **options): + ignore_errors = options.get('ignore_errors') + remove_duplicates = options.get('remove_duplicates') + prune = options.get('prune') + set_uuid = ast.literal_eval(options.get('set_uuid', 'False')) + set_attrib = ast.literal_eval(options.get('set_attrib', 'True')) + set_links = ast.literal_eval(options.get('set_links', 'True')) + + delete_orphaned_thumbnails = options.get('delete_orphaned_thumbnails') + filter = options.get('filter') + if not options.get('username'): + username = None + else: + username = options.get('username') + + all_datasets = Dataset.objects.all().order_by('name') + if filter: + all_datasets = all_datasets.filter(name__icontains=filter) + if username: + all_datasets = all_datasets.filter(owner__username=username) + + for index, layer in enumerate(all_datasets): + print(f"[{(index + 1)} / {len(all_datasets)}] Updating Dataset [{layer.name}] ...") + try: + # recalculate the layer statistics + if set_attrib: + set_attributes(layer, overwrite=True) + + if set_uuid and hasattr(settings, 'LAYER_UUID_HANDLER') and settings.LAYER_UUID_HANDLER != '': + from geonode.layers.utils import get_uuid_handler + uuid = get_uuid_handler()(layer).create_uuid() + la = Dataset.objects.filter(resourcebase_ptr=layer.resourcebase_ptr) + la.update(uuid=uuid) + layer.refresh_from_db() + # refresh metadata links + + if set_links: + set_resource_default_links(layer, layer, prune=prune) + + # refresh catalogue metadata records + catalogue_post_save(instance=layer, sender=layer.__class__) + + # remove duplicates + if remove_duplicates: + remove_duplicate_links(layer) + except Exception as e: + import traceback + traceback.print_exc() + if ignore_errors: + logger.error(f"[ERROR] Dataset [{layer.name}] couldn't be updated") + else: + raise e + + # delete orphaned thumbs + if delete_orphaned_thumbnails: + delete_orphaned_thumbs() diff --git a/geonode/base/management/commands/squashmigrations.py b/geonode/base/management/commands/squashmigrations.py new file mode 100644 index 0000000..26d2131 --- /dev/null +++ b/geonode/base/management/commands/squashmigrations.py @@ -0,0 +1,41 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +""" +A squashmigrations command which does some monkeypatching on the RunPython operation +to allow squashing migrations which use RunPython.noop, even on Python 2 +""" +from django.core.management.commands.squashmigrations import Command as DjangoCommand +from django.db.migrations import RunPython + + +# A staticmethod decorator is needed (out of a class!) because we are going to +# monkeypatch this function into a class and it will be introspected by code that +# will think it is an unbound instance method without the decorator. +@staticmethod +def noop(apps, schema_editor): + return None + + +class Command(DjangoCommand): + + def handle(self, **options): + # Monkeypatch + RunPython.noop = noop + # Invoke original + return super().handle(**options) diff --git a/geonode/base/middleware.py b/geonode/base/middleware.py new file mode 100644 index 0000000..e1ac3fd --- /dev/null +++ b/geonode/base/middleware.py @@ -0,0 +1,97 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + + +# Geonode functionality + +from django.shortcuts import render +from geonode.base.utils import configuration_session_cache + + +class ReadOnlyMiddleware: + """ + A Middleware disabling all content modifying requests, if read-only Configuration setting is True, + with an exception for whitelisted url names. + """ + + FORBIDDEN_HTTP_METHODS = ["POST", "PUT", "DELETE"] + + WHITELISTED_URL_NAMES = [ + "login", + "logout", + "account_login", + "account_logout", + "ows_endpoint", + ] + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + return self.get_response(request) + + def process_view(self, request, view_func, view_args, view_kwargs): + session = request.session + configuration_session_cache(session) + + # check if the Geonode instance is read-only + if session.get("config").get("configuration").get("read_only"): + # allow superadmin users to do whatever they want + if not request.user.is_superuser or not request.user.is_active: + # check if the request's method is forbidden in read-only instance + if request.method in self.FORBIDDEN_HTTP_METHODS: + # check if the request is not against whitelisted views (check by URL names) + if request.resolver_match.url_name not in self.WHITELISTED_URL_NAMES: + # return HttpResponse('Error: Instance in read-only mode', status=405) + return render(request, "base/read_only_violation.html", status=405) + + +class MaintenanceMiddleware: + """ + A Middleware redirecting all requests to maintenance info page, except: + - admin panel login, + - admin panel logout, + - requests performed by superuser, + if maintenance Configuration setting is True. + """ + + # URL's enabling superuser to login/logout to/from admin panel + WHITELISTED_URL_NAMES = [ + "login", + "logout", + "index", + ] + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + return self.get_response(request) + + def process_view(self, request, view_func, view_args, view_kwargs): + session = request.session + configuration_session_cache(session) + + # check if the Geonode instance is in maintenance mode + if session.get("config").get("configuration").get("maintenance"): + # allow superadmin users to do whatever they want + if not request.user.is_superuser: + # check if the request is not against whitelisted views (check by URL names) + if request.resolver_match.url_name not in self.WHITELISTED_URL_NAMES: + return render(request, "base/maintenance.html", status=503) diff --git a/geonode/base/migrations/0027_auto_20170801_1228.py b/geonode/base/migrations/0027_auto_20170801_1228.py new file mode 100644 index 0000000..1f08a61 --- /dev/null +++ b/geonode/base/migrations/0027_auto_20170801_1228.py @@ -0,0 +1,31 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '26_to_27'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='abstract', + field=models.TextField(help_text='brief narrative summary of the content of the resource(s)', max_length=2000, verbose_name='abstract', blank=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='data_quality_statement', + field=models.TextField(help_text="general explanation of the data producer's knowledge about the lineage of a dataset", max_length=2000, null=True, verbose_name='data quality statement', blank=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='purpose', + field=models.TextField(help_text='summary of the intentions with which the resource(s) was developed', max_length=500, null=True, verbose_name='purpose', blank=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='supplemental_information', + field=models.TextField(default='No information provided', help_text='any other descriptive information about the dataset', max_length=2000, verbose_name='supplemental information'), + ), + ] diff --git a/geonode/base/migrations/0027_auto_20170801_1228_squashed_0037_auto_20190222_1347.py b/geonode/base/migrations/0027_auto_20170801_1228_squashed_0037_auto_20190222_1347.py new file mode 100644 index 0000000..f0acc20 --- /dev/null +++ b/geonode/base/migrations/0027_auto_20170801_1228_squashed_0037_auto_20190222_1347.py @@ -0,0 +1,187 @@ +# Generated by Django 1.11.20 on 2019-04-04 08:29 + + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('base', '0027_auto_20170801_1228'), ('base', '0028_resourcebase_is_approved'), ('base', '0029_auto_20171114_0341'), ('base', '0030_auto_20180309_0833'), ('base', '0031_auto_20180309_0837'), ('base', '0032_auto_20180329_1844'), ('base', '0033_auto_20180330_0951'), ('base', '0034_auto_20180606_1543'), ('base', '0035_resourcebase_dirty_state'), ('base', '0036_auto_20190129_1433'), ('base', '0037_auto_20190222_1347')] + + dependencies = [ + ('base', '26_to_27'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='abstract', + field=models.TextField(blank=True, help_text='brief narrative summary of the content of the resource(s)', max_length=2000, verbose_name='abstract'), + ), + migrations.AlterField( + model_name='resourcebase', + name='data_quality_statement', + field=models.TextField(blank=True, help_text="general explanation of the data producer's knowledge about the lineage of a dataset", max_length=2000, null=True, verbose_name='data quality statement'), + ), + migrations.AlterField( + model_name='resourcebase', + name='purpose', + field=models.TextField(blank=True, help_text='summary of the intentions with which the resource(s) was developed', max_length=500, null=True, verbose_name='purpose'), + ), + migrations.AlterField( + model_name='resourcebase', + name='supplemental_information', + field=models.TextField(default='No information provided', help_text='any other descriptive information about the dataset', max_length=2000, verbose_name='supplemental information'), + ), + migrations.AddField( + model_name='resourcebase', + name='is_approved', + field=models.BooleanField(default=True, help_text='Is this resource validated from a publisher or editor?', verbose_name='Approved'), + ), + migrations.AlterField( + model_name='resourcebase', + name='language', + field=models.CharField(choices=[('abk','Abkhazian'), ('aar','Afar'), ('afr','Afrikaans'), ('amh','Amharic'), ('ara','Arabic'), ('asm','Assamese'), ('aym','Aymara'), ('aze','Azerbaijani'), ('bak','Bashkir'), ('ben','Bengali'), ('bih','Bihari'), ('bis','Bislama'), ('bre','Breton'), ('bul','Bulgarian'), ('bel','Byelorussian'), ('cat','Catalan'), ('cos','Corsican'), ('dan','Danish'), ('dzo','Dzongkha'), ('eng','English'), ('fra','French'), ('epo','Esperanto'), ('est','Estonian'), ('fao','Faroese'), ('fij','Fijian'), ('fin','Finnish'), ('fry','Frisian'), ('glg','Gallegan'), ('ger','German'), ('gre','Greek'), ('kal','Greenlandic'), ('grn','Guarani'), ('guj','Gujarati'), ('hau','Hausa'), ('heb','Hebrew'), ('hin','Hindi'), ('hun','Hungarian'), ('ind','Indonesian'), ('ina','Interlingua (International Auxiliary language Association)'), ('iku','Inuktitut'), ('ipk','Inupiak'), ('ita','Italian'), ('jpn','Japanese'), ('kan','Kannada'), ('kas','Kashmiri'), ('kaz','Kazakh'), ('khm','Khmer'), ('kin','Kinyarwanda'), ('kir','Kirghiz'), ('kor','Korean'), ('kur','Kurdish'), ('oci', b"Langue d 'Oc (post 1500)"), ('lao','Lao'), ('lat','Latin'), ('lav','Latvian'), ('lin','Lingala'), ('lit','Lithuanian'), ('mlg','Malagasy'), ('mlt','Maltese'), ('mar','Marathi'), ('mol','Moldavian'), ('mon','Mongolian'), ('nau','Nauru'), ('nep','Nepali'), ('nor','Norwegian'), ('ori','Oriya'), ('orm','Oromo'), ('pan','Panjabi'), ('pol','Polish'), ('por','Portuguese'), ('pus','Pushto'), ('que','Quechua'), ('roh','Rhaeto-Romance'), ('run','Rundi'), ('rus','Russian'), ('smo','Samoan'), ('sag','Sango'), ('san','Sanskrit'), ('scr','Serbo-Croatian'), ('sna','Shona'), ('snd','Sindhi'), ('sin','Singhalese'), ('ssw','Siswant'), ('slv','Slovenian'), ('som','Somali'), ('sot','Sotho'), ('spa','Spanish'), ('sun','Sudanese'), ('swa','Swahili'), ('tgl','Tagalog'), ('tgk','Tajik'), ('tam','Tamil'), ('tat','Tatar'), ('tel','Telugu'), ('tha','Thai'), ('tir','Tigrinya'), ('tog','Tonga (Nyasa)'), ('tso','Tsonga'), ('tsn','Tswana'), ('tur','Turkish'), ('tuk','Turkmen'), ('twi','Twi'), ('uig','Uighur'), ('ukr','Ukrainian'), ('urd','Urdu'), ('uzb','Uzbek'), ('vie','Vietnamese'), ('vol','Volap\xc3\xbck'), ('wol','Wolof'), ('xho','Xhosa'), ('yid','Yiddish'), ('yor','Yoruba'), ('zha','Zhuang'), ('zul','Zulu')], default='eng', help_text='language used within the dataset', max_length=3, verbose_name='language'), + ), + migrations.AlterField( + model_name='region', + name='bbox_x0', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_x1', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y0', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y1', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_x0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_x1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_x0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_x1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_y0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_y1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='srid', + field=models.CharField(default='EPSG:4326', max_length=30), + ), + migrations.AlterField( + model_name='resourcebase', + name='srid', + field=models.CharField(default='EPSG:4326', max_length=30), + ), + migrations.AlterField( + model_name='resourcebase', + name='metadata_uploaded_preserve', + field=models.BooleanField(default=False, verbose_name='Metadata uploaded preserve'), + ), + migrations.AlterField( + model_name='resourcebase', + name='thumbnail_url', + field=models.TextField(blank=True, null=True, verbose_name='Thumbnail url'), + ), + migrations.AlterField( + model_name='resourcebase', + name='tkeywords', + field=models.ManyToManyField(blank=True, help_text='formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject (space or comma-separated', to='base.ThesaurusKeyword', verbose_name='keywords'), + ), + migrations.AddField( + model_name='resourcebase', + name='dirty_state', + field=models.BooleanField(default=False, help_text='Security Rules Are Not Synched with GeoServer!', verbose_name='Dirty State'), + ), + migrations.CreateModel( + name='Menu', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('order', models.IntegerField()), + ], + options={ + 'ordering': ['order'], + }, + ), + migrations.CreateModel( + name='MenuItem', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('order', models.IntegerField()), + ('blank_target', models.BooleanField()), + ('url', models.CharField(max_length=2000)), + ('menu', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.Menu')), + ], + options={ + 'ordering': ['order'], + }, + ), + migrations.CreateModel( + name='MenuPlaceholder', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ], + ), + migrations.AddField( + model_name='menu', + name='placeholder', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.MenuPlaceholder'), + ), + migrations.AlterUniqueTogether( + name='menuitem', + unique_together={('menu', 'order'), ('menu', 'title')}, + ), + migrations.AlterUniqueTogether( + name='menu', + unique_together={('placeholder', 'order'), ('placeholder', 'title')}, + ), + migrations.AlterField( + model_name='resourcebase', + name='language', + field=models.CharField(choices=[('abk','Abkhazian'), ('aar','Afar'), ('afr','Afrikaans'), ('amh','Amharic'), ('ara','Arabic'), ('asm','Assamese'), ('aym','Aymara'), ('aze','Azerbaijani'), ('bak','Bashkir'), ('ben','Bengali'), ('bih','Bihari'), ('bis','Bislama'), ('bre','Breton'), ('bul','Bulgarian'), ('bel','Byelorussian'), ('cat','Catalan'), ('chi','Chinese'), ('cos','Corsican'), ('dan','Danish'), ('dzo','Dzongkha'), ('eng','English'), ('fra','French'), ('epo','Esperanto'), ('est','Estonian'), ('fao','Faroese'), ('fij','Fijian'), ('fin','Finnish'), ('fry','Frisian'), ('glg','Gallegan'), ('ger','German'), ('gre','Greek'), ('kal','Greenlandic'), ('grn','Guarani'), ('guj','Gujarati'), ('hau','Hausa'), ('heb','Hebrew'), ('hin','Hindi'), ('hun','Hungarian'), ('ind','Indonesian'), ('ina','Interlingua (International Auxiliary language Association)'), ('iku','Inuktitut'), ('ipk','Inupiak'), ('ita','Italian'), ('jpn','Japanese'), ('kan','Kannada'), ('kas','Kashmiri'), ('kaz','Kazakh'), ('khm','Khmer'), ('kin','Kinyarwanda'), ('kir','Kirghiz'), ('kor','Korean'), ('kur','Kurdish'), ('oci', b"Langue d 'Oc (post 1500)"), ('lao','Lao'), ('lat','Latin'), ('lav','Latvian'), ('lin','Lingala'), ('lit','Lithuanian'), ('mlg','Malagasy'), ('mlt','Maltese'), ('mar','Marathi'), ('mol','Moldavian'), ('mon','Mongolian'), ('nau','Nauru'), ('nep','Nepali'), ('nor','Norwegian'), ('ori','Oriya'), ('orm','Oromo'), ('pan','Panjabi'), ('pol','Polish'), ('por','Portuguese'), ('pus','Pushto'), ('que','Quechua'), ('roh','Rhaeto-Romance'), ('run','Rundi'), ('rus','Russian'), ('smo','Samoan'), ('sag','Sango'), ('san','Sanskrit'), ('scr','Serbo-Croatian'), ('sna','Shona'), ('snd','Sindhi'), ('sin','Singhalese'), ('ssw','Siswant'), ('slv','Slovenian'), ('som','Somali'), ('sot','Sotho'), ('spa','Spanish'), ('sun','Sudanese'), ('swa','Swahili'), ('tgl','Tagalog'), ('tgk','Tajik'), ('tam','Tamil'), ('tat','Tatar'), ('tel','Telugu'), ('tha','Thai'), ('tir','Tigrinya'), ('tog','Tonga (Nyasa)'), ('tso','Tsonga'), ('tsn','Tswana'), ('tur','Turkish'), ('tuk','Turkmen'), ('twi','Twi'), ('uig','Uighur'), ('ukr','Ukrainian'), ('urd','Urdu'), ('uzb','Uzbek'), ('vie','Vietnamese'), ('vol','Volap\xc3\xbck'), ('wol','Wolof'), ('xho','Xhosa'), ('yid','Yiddish'), ('yor','Yoruba'), ('zha','Zhuang'), ('zul','Zulu')], default='eng', help_text='language used within the dataset', max_length=3, verbose_name='language'), + ), + ] diff --git a/geonode/base/migrations/0028_curatedthumbnail.py b/geonode/base/migrations/0028_curatedthumbnail.py new file mode 100644 index 0000000..b894d1e --- /dev/null +++ b/geonode/base/migrations/0028_curatedthumbnail.py @@ -0,0 +1,23 @@ +# Generated by Django 1.11.23 on 2019-08-14 14:10 + + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0027_auto_20170801_1228_squashed_0037_auto_20190222_1347'), + ] + + operations = [ + migrations.CreateModel( + name='CuratedThumbnail', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('img', models.ImageField(upload_to='curated_thumbs')), + ('resource', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase')), + ], + ), + ] diff --git a/geonode/base/migrations/0028_resourcebase_is_approved.py b/geonode/base/migrations/0028_resourcebase_is_approved.py new file mode 100644 index 0000000..b0c5445 --- /dev/null +++ b/geonode/base/migrations/0028_resourcebase_is_approved.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0027_auto_20170801_1228'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='is_approved', + field=models.BooleanField(default=False, help_text='Is this resource validated from a publisher or editor?', verbose_name='Approved'), + ), + ] diff --git a/geonode/base/migrations/0029_auto_20171114_0341.py b/geonode/base/migrations/0029_auto_20171114_0341.py new file mode 100644 index 0000000..26dc3aa --- /dev/null +++ b/geonode/base/migrations/0029_auto_20171114_0341.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0028_resourcebase_is_approved'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='language', + field=models.CharField(default='eng', help_text='language used within the dataset', max_length=3, verbose_name='language', choices=[('abk','Abkhazian'), ('aar','Afar'), ('afr','Afrikaans'), ('amh','Amharic'), ('ara','Arabic'), ('asm','Assamese'), ('aym','Aymara'), ('aze','Azerbaijani'), ('bak','Bashkir'), ('ben','Bengali'), ('bih','Bihari'), ('bis','Bislama'), ('bre','Breton'), ('bul','Bulgarian'), ('bel','Byelorussian'), ('cat','Catalan'), ('cos','Corsican'), ('dan','Danish'), ('dzo','Dzongkha'), ('eng','English'), ('fra','French'), ('epo','Esperanto'), ('est','Estonian'), ('fao','Faroese'), ('fij','Fijian'), ('fin','Finnish'), ('fry','Frisian'), ('glg','Gallegan'), ('ger','German'), ('gre','Greek'), ('kal','Greenlandic'), ('grn','Guarani'), ('guj','Gujarati'), ('hau','Hausa'), ('heb','Hebrew'), ('hin','Hindi'), ('hun','Hungarian'), ('ind','Indonesian'), ('ina','Interlingua (International Auxiliary language Association)'), ('iku','Inuktitut'), ('ipk','Inupiak'), ('ita','Italian'), ('jpn','Japanese'), ('kan','Kannada'), ('kas','Kashmiri'), ('kaz','Kazakh'), ('khm','Khmer'), ('kin','Kinyarwanda'), ('kir','Kirghiz'), ('kor','Korean'), ('kur','Kurdish'), ('oci', b"Langue d 'Oc (post 1500)"), ('lao','Lao'), ('lat','Latin'), ('lav','Latvian'), ('lin','Lingala'), ('lit','Lithuanian'), ('mlg','Malagasy'), ('mlt','Maltese'), ('mar','Marathi'), ('mol','Moldavian'), ('mon','Mongolian'), ('nau','Nauru'), ('nep','Nepali'), ('nor','Norwegian'), ('ori','Oriya'), ('orm','Oromo'), ('pan','Panjabi'), ('pol','Polish'), ('por','Portuguese'), ('pus','Pushto'), ('que','Quechua'), ('roh','Rhaeto-Romance'), ('run','Rundi'), ('rus','Russian'), ('smo','Samoan'), ('sag','Sango'), ('san','Sanskrit'), ('scr','Serbo-Croatian'), ('sna','Shona'), ('snd','Sindhi'), ('sin','Singhalese'), ('ssw','Siswant'), ('slv','Slovenian'), ('som','Somali'), ('sot','Sotho'), ('spa','Spanish'), ('sun','Sudanese'), ('swa','Swahili'), ('tgl','Tagalog'), ('tgk','Tajik'), ('tam','Tamil'), ('tat','Tatar'), ('tel','Telugu'), ('tha','Thai'), ('tir','Tigrinya'), ('tog','Tonga (Nyasa)'), ('tso','Tsonga'), ('tsn','Tswana'), ('tur','Turkish'), ('tuk','Turkmen'), ('twi','Twi'), ('uig','Uighur'), ('ukr','Ukrainian'), ('urd','Urdu'), ('uzb','Uzbek'), ('vie','Vietnamese'), ('vol','Volap\xc3\xbck'), ('wol','Wolof'), ('xho','Xhosa'), ('yid','Yiddish'), ('yor','Yoruba'), ('zha','Zhuang'), ('zul','Zulu')]), + ), + ] diff --git a/geonode/base/migrations/0029_resourcebase_last_updated.py b/geonode/base/migrations/0029_resourcebase_last_updated.py new file mode 100644 index 0000000..1f0e512 --- /dev/null +++ b/geonode/base/migrations/0029_resourcebase_last_updated.py @@ -0,0 +1,20 @@ +# Generated by Django 1.11.25 on 2019-10-04 15:48 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0028_curatedthumbnail'), + ('services', '0028_remove_service_last_updated'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='last_updated', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/geonode/base/migrations/0030_auto_20180309_0833.py b/geonode/base/migrations/0030_auto_20180309_0833.py new file mode 100644 index 0000000..de9d98b --- /dev/null +++ b/geonode/base/migrations/0030_auto_20180309_0833.py @@ -0,0 +1,34 @@ +# Generated by Django 1.9.13 on 2018-03-09 08:33 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0029_auto_20171114_0341'), + ] + + operations = [ + migrations.AlterField( + model_name='region', + name='bbox_x0', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_x1', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y0', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y1', + field=models.DecimalField(blank=True, decimal_places=10, max_digits=30, null=True), + ), + ] diff --git a/geonode/base/migrations/0030_resourcebase_created.py b/geonode/base/migrations/0030_resourcebase_created.py new file mode 100644 index 0000000..dcb5153 --- /dev/null +++ b/geonode/base/migrations/0030_resourcebase_created.py @@ -0,0 +1,22 @@ +# Generated by Django 1.11.25 on 2019-10-04 15:59 + + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0029_resourcebase_last_updated'), + ('services', '0029_remove_service_created'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + ] diff --git a/geonode/base/migrations/0031_auto_20180309_0837.py b/geonode/base/migrations/0031_auto_20180309_0837.py new file mode 100644 index 0000000..2650819 --- /dev/null +++ b/geonode/base/migrations/0031_auto_20180309_0837.py @@ -0,0 +1,54 @@ +# Generated by Django 1.9.13 on 2018-03-09 08:37 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0030_auto_20180309_0833'), + ] + + operations = [ + migrations.AlterField( + model_name='region', + name='bbox_x0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_x1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='region', + name='bbox_y1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_x0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_x1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_y0', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='bbox_y1', + field=models.DecimalField(blank=True, decimal_places=15, max_digits=30, null=True), + ), + ] diff --git a/geonode/base/migrations/0031_auto_20200114_1651.py b/geonode/base/migrations/0031_auto_20200114_1651.py new file mode 100644 index 0000000..b13f98a --- /dev/null +++ b/geonode/base/migrations/0031_auto_20200114_1651.py @@ -0,0 +1,19 @@ +# Generated by Django 1.11.27 on 2020-01-14 16:51 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0030_resourcebase_created'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='language', + field=models.CharField(choices=[('abk', 'Abkhazian'), ('aar', 'Afar'), ('afr', 'Afrikaans'), ('amh', 'Amharic'), ('ara', 'Arabic'), ('asm', 'Assamese'), ('aym', 'Aymara'), ('aze', 'Azerbaijani'), ('bak', 'Bashkir'), ('ben', 'Bengali'), ('bih', 'Bihari'), ('bis', 'Bislama'), ('bre', 'Breton'), ('bul', 'Bulgarian'), ('bel', 'Byelorussian'), ('cat', 'Catalan'), ('chi', 'Chinese'), ('cos', 'Corsican'), ('dan', 'Danish'), ('dzo', 'Dzongkha'), ('eng', 'English'), ('fra', 'French'), ('epo', 'Esperanto'), ('est', 'Estonian'), ('fao', 'Faroese'), ('fij', 'Fijian'), ('fin', 'Finnish'), ('fry', 'Frisian'), ('glg', 'Gallegan'), ('ger', 'German'), ('gre', 'Greek'), ('kal', 'Greenlandic'), ('grn', 'Guarani'), ('guj', 'Gujarati'), ('hau', 'Hausa'), ('heb', 'Hebrew'), ('hin', 'Hindi'), ('hun', 'Hungarian'), ('ind', 'Indonesian'), ('ina', 'Interlingua (International Auxiliary language Association)'), ('iku', 'Inuktitut'), ('ipk', 'Inupiak'), ('ita', 'Italian'), ('jpn', 'Japanese'), ('kan', 'Kannada'), ('kas', 'Kashmiri'), ('kaz', 'Kazakh'), ('khm', 'Khmer'), ('kin', 'Kinyarwanda'), ('kir', 'Kirghiz'), ('kor', 'Korean'), ('kur', 'Kurdish'), ('oci', "Langue d 'Oc (post 1500)"), ('lao', 'Lao'), ('lat', 'Latin'), ('lav', 'Latvian'), ('lin', 'Lingala'), ('lit', 'Lithuanian'), ('mlg', 'Malagasy'), ('mlt', 'Maltese'), ('mar', 'Marathi'), ('mol', 'Moldavian'), ('mon', 'Mongolian'), ('nau', 'Nauru'), ('nep', 'Nepali'), ('nor', 'Norwegian'), ('ori', 'Oriya'), ('orm', 'Oromo'), ('pan', 'Panjabi'), ('pol', 'Polish'), ('por', 'Portuguese'), ('pus', 'Pushto'), ('que', 'Quechua'), ('roh', 'Rhaeto-Romance'), ('run', 'Rundi'), ('rus', 'Russian'), ('smo', 'Samoan'), ('sag', 'Sango'), ('san', 'Sanskrit'), ('scr', 'Serbo-Croatian'), ('sna', 'Shona'), ('snd', 'Sindhi'), ('sin', 'Singhalese'), ('ssw', 'Siswant'), ('slv', 'Slovenian'), ('som', 'Somali'), ('sot', 'Sotho'), ('spa', 'Spanish'), ('sun', 'Sudanese'), ('swa', 'Swahili'), ('tgl', 'Tagalog'), ('tgk', 'Tajik'), ('tam', 'Tamil'), ('tat', 'Tatar'), ('tel', 'Telugu'), ('tha', 'Thai'), ('tir', 'Tigrinya'), ('tog', 'Tonga (Nyasa)'), ('tso', 'Tsonga'), ('tsn', 'Tswana'), ('tur', 'Turkish'), ('tuk', 'Turkmen'), ('twi', 'Twi'), ('uig', 'Uighur'), ('ukr', 'Ukrainian'), ('urd', 'Urdu'), ('uzb', 'Uzbek'), ('vie', 'Vietnamese'), ('vol', 'Volapük'), ('wol', 'Wolof'), ('xho', 'Xhosa'), ('yid', 'Yiddish'), ('yor', 'Yoruba'), ('zha', 'Zhuang'), ('zul', 'Zulu')], default='eng', help_text='language used within the dataset', max_length=3, verbose_name='language'), + ), + ] diff --git a/geonode/base/migrations/0032_auto_20180329_1844.py b/geonode/base/migrations/0032_auto_20180329_1844.py new file mode 100644 index 0000000..6a4e039 --- /dev/null +++ b/geonode/base/migrations/0032_auto_20180329_1844.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0031_auto_20180309_0837'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='is_approved', + field=models.BooleanField(default=True, help_text='Is this resource validated from a publisher or editor?', verbose_name='Approved'), + ), + ] diff --git a/geonode/base/migrations/0032_auto_20200115_1121.py b/geonode/base/migrations/0032_auto_20200115_1121.py new file mode 100644 index 0000000..b7c8129 --- /dev/null +++ b/geonode/base/migrations/0032_auto_20200115_1121.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.9 on 2020-01-15 11:21 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0031_auto_20200114_1651'), + ] + + operations = [ + migrations.AlterField( + model_name='curatedthumbnail', + name='resource', + field=models.OneToOneField(on_delete=models.CASCADE, to='base.ResourceBase'), + ), + migrations.AlterField( + model_name='resourcebase', + name='category', + field=models.ForeignKey(blank=True, help_text='high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets.', limit_choices_to=models.Q(is_choice=True), null=True, on_delete=django.db.models.deletion.CASCADE, to='base.TopicCategory'), + ), + migrations.AlterField( + model_name='resourcebase', + name='restriction_code_type', + field=models.ForeignKey(blank=True, help_text='limitation(s) placed upon the access or use of the data.', limit_choices_to=models.Q(is_choice=True), null=True, on_delete=django.db.models.deletion.CASCADE, to='base.RestrictionCodeType', verbose_name='restrictions'), + ), + migrations.AlterField( + model_name='resourcebase', + name='spatial_representation_type', + field=models.ForeignKey(blank=True, help_text='method used to represent geographic information in the dataset.', limit_choices_to=models.Q(is_choice=True), null=True, on_delete=django.db.models.deletion.CASCADE, to='base.SpatialRepresentationType', verbose_name='spatial representation type'), + ), + ] diff --git a/geonode/base/migrations/0033_auto_20180330_0951.py b/geonode/base/migrations/0033_auto_20180330_0951.py new file mode 100644 index 0000000..6979bbe --- /dev/null +++ b/geonode/base/migrations/0033_auto_20180330_0951.py @@ -0,0 +1,21 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0032_auto_20180329_1844'), + ] + + operations = [ + migrations.AlterField( + model_name='region', + name='srid', + field=models.CharField(default='EPSG:4326', max_length=30), + ), + migrations.AlterField( + model_name='resourcebase', + name='srid', + field=models.CharField(default='EPSG:4326', max_length=30), + ), + ] diff --git a/geonode/base/migrations/0033_auto_20200115_1145.py b/geonode/base/migrations/0033_auto_20200115_1145.py new file mode 100644 index 0000000..c794b49 --- /dev/null +++ b/geonode/base/migrations/0033_auto_20200115_1145.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.9 on 2020-01-15 11:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0032_auto_20200115_1121'), + ] + + operations = [ + migrations.AlterModelOptions( + name='resourcebase', + options={'permissions': (('change_resourcebase_permissions', 'Can change resource permissions'), ('download_resourcebase', 'Can download resource'), ('publish_resourcebase', 'Can publish resource'), ('change_resourcebase_metadata', 'Can change resource metadata'))}, + ), + ] diff --git a/geonode/base/migrations/0034_auto_20180606_1543.py b/geonode/base/migrations/0034_auto_20180606_1543.py new file mode 100644 index 0000000..b572a2d --- /dev/null +++ b/geonode/base/migrations/0034_auto_20180606_1543.py @@ -0,0 +1,29 @@ +# Generated by Django 1.11.11 on 2018-06-06 15:43 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0033_auto_20180330_0951'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='metadata_uploaded_preserve', + field=models.BooleanField(default=False, verbose_name='Metadata uploaded preserve'), + ), + migrations.AlterField( + model_name='resourcebase', + name='thumbnail_url', + field=models.TextField(blank=True, null=True, verbose_name='Thumbnail url'), + ), + migrations.AlterField( + model_name='resourcebase', + name='tkeywords', + field=models.ManyToManyField(blank=True, help_text='formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject (space or comma-separated', to='base.ThesaurusKeyword', verbose_name='keywords'), + ), + ] diff --git a/geonode/base/migrations/0034_auto_20200212_0410.py b/geonode/base/migrations/0034_auto_20200212_0410.py new file mode 100644 index 0000000..867aa59 --- /dev/null +++ b/geonode/base/migrations/0034_auto_20200212_0410.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.10 on 2020-02-12 04:10 + +from django.db import migrations +from geonode.base.models import ContactRole + +def forwards_func(apps, schema_editor): + ContactRole.objects.filter(resource_id__isnull=True).delete() + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0033_auto_20200115_1145'), + ] + + operations = [ + migrations.RunPython(forwards_func, migrations.RunPython.noop) + ] diff --git a/geonode/base/migrations/0035_auto_20200212_0428.py b/geonode/base/migrations/0035_auto_20200212_0428.py new file mode 100644 index 0000000..46dc9f3 --- /dev/null +++ b/geonode/base/migrations/0035_auto_20200212_0428.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.10 on 2020-02-12 04:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0034_auto_20200212_0410'), + ] + + operations = [ + migrations.AlterField( + model_name='contactrole', + name='resource', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase'), + ), + migrations.AlterField( + model_name='region', + name='level', + field=models.PositiveIntegerField(editable=False), + ), + migrations.AlterField( + model_name='region', + name='lft', + field=models.PositiveIntegerField(editable=False), + ), + migrations.AlterField( + model_name='region', + name='rght', + field=models.PositiveIntegerField(editable=False), + ), + ] diff --git a/geonode/base/migrations/0035_resourcebase_dirty_state.py b/geonode/base/migrations/0035_resourcebase_dirty_state.py new file mode 100644 index 0000000..ad52f2d --- /dev/null +++ b/geonode/base/migrations/0035_resourcebase_dirty_state.py @@ -0,0 +1,19 @@ +# Generated by Django 1.11.11 on 2018-10-02 15:42 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0034_auto_20180606_1543'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='dirty_state', + field=models.BooleanField(default=False, help_text='Security Rules Are Not Synched with GeoServer!', verbose_name='Dirty State'), + ), + ] diff --git a/geonode/base/migrations/0036_auto_20190129_1433.py b/geonode/base/migrations/0036_auto_20190129_1433.py new file mode 100644 index 0000000..afe3745 --- /dev/null +++ b/geonode/base/migrations/0036_auto_20190129_1433.py @@ -0,0 +1,60 @@ +# Generated by Django 1.11.16 on 2019-01-29 14:33 + + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0035_resourcebase_dirty_state'), + ] + + operations = [ + migrations.CreateModel( + name='Menu', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('order', models.IntegerField()), + ], + options={ + 'ordering': ['order'], + }, + ), + migrations.CreateModel( + name='MenuItem', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('order', models.IntegerField()), + ('blank_target', models.BooleanField()), + ('url', models.CharField(max_length=2000)), + ('menu', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.Menu')), + ], + options={ + 'ordering': ['order'], + }, + ), + migrations.CreateModel( + name='MenuPlaceholder', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ], + ), + migrations.AddField( + model_name='menu', + name='placeholder', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.MenuPlaceholder'), + ), + migrations.AlterUniqueTogether( + name='menuitem', + unique_together={('menu', 'order'), ('menu', 'title')}, + ), + migrations.AlterUniqueTogether( + name='menu', + unique_together={('placeholder', 'order'), ('placeholder', 'title')}, + ), + ] diff --git a/geonode/base/migrations/0036_resourcebase_doi.py b/geonode/base/migrations/0036_resourcebase_doi.py new file mode 100644 index 0000000..946d474 --- /dev/null +++ b/geonode/base/migrations/0036_resourcebase_doi.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.10 on 2020-03-03 11:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0035_auto_20200212_0428'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='doi', + field=models.TextField(blank=True, help_text='a DOI will be added by Admin before publication.', null=True, verbose_name='DOI'), + ), + ] diff --git a/geonode/base/migrations/0037_auto_20190222_1347.py b/geonode/base/migrations/0037_auto_20190222_1347.py new file mode 100644 index 0000000..00ddf66 --- /dev/null +++ b/geonode/base/migrations/0037_auto_20190222_1347.py @@ -0,0 +1,19 @@ +# Generated by Django 1.11.20 on 2019-02-22 13:47 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0036_auto_20190129_1433'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='language', + field=models.CharField(choices=[('abk','Abkhazian'), ('aar','Afar'), ('afr','Afrikaans'), ('amh','Amharic'), ('ara','Arabic'), ('asm','Assamese'), ('aym','Aymara'), ('aze','Azerbaijani'), ('bak','Bashkir'), ('ben','Bengali'), ('bih','Bihari'), ('bis','Bislama'), ('bre','Breton'), ('bul','Bulgarian'), ('bel','Byelorussian'), ('cat','Catalan'), ('chi','Chinese'), ('cos','Corsican'), ('dan','Danish'), ('dzo','Dzongkha'), ('eng','English'), ('fra','French'), ('epo','Esperanto'), ('est','Estonian'), ('fao','Faroese'), ('fij','Fijian'), ('fin','Finnish'), ('fry','Frisian'), ('glg','Gallegan'), ('ger','German'), ('gre','Greek'), ('kal','Greenlandic'), ('grn','Guarani'), ('guj','Gujarati'), ('hau','Hausa'), ('heb','Hebrew'), ('hin','Hindi'), ('hun','Hungarian'), ('ind','Indonesian'), ('ina','Interlingua (International Auxiliary language Association)'), ('iku','Inuktitut'), ('ipk','Inupiak'), ('ita','Italian'), ('jpn','Japanese'), ('kan','Kannada'), ('kas','Kashmiri'), ('kaz','Kazakh'), ('khm','Khmer'), ('kin','Kinyarwanda'), ('kir','Kirghiz'), ('kor','Korean'), ('kur','Kurdish'), ('oci', b"Langue d 'Oc (post 1500)"), ('lao','Lao'), ('lat','Latin'), ('lav','Latvian'), ('lin','Lingala'), ('lit','Lithuanian'), ('mlg','Malagasy'), ('mlt','Maltese'), ('mar','Marathi'), ('mol','Moldavian'), ('mon','Mongolian'), ('nau','Nauru'), ('nep','Nepali'), ('nor','Norwegian'), ('ori','Oriya'), ('orm','Oromo'), ('pan','Panjabi'), ('pol','Polish'), ('por','Portuguese'), ('pus','Pushto'), ('que','Quechua'), ('roh','Rhaeto-Romance'), ('run','Rundi'), ('rus','Russian'), ('smo','Samoan'), ('sag','Sango'), ('san','Sanskrit'), ('scr','Serbo-Croatian'), ('sna','Shona'), ('snd','Sindhi'), ('sin','Singhalese'), ('ssw','Siswant'), ('slv','Slovenian'), ('som','Somali'), ('sot','Sotho'), ('spa','Spanish'), ('sun','Sudanese'), ('swa','Swahili'), ('tgl','Tagalog'), ('tgk','Tajik'), ('tam','Tamil'), ('tat','Tatar'), ('tel','Telugu'), ('tha','Thai'), ('tir','Tigrinya'), ('tog','Tonga (Nyasa)'), ('tso','Tsonga'), ('tsn','Tswana'), ('tur','Turkish'), ('tuk','Turkmen'), ('twi','Twi'), ('uig','Uighur'), ('ukr','Ukrainian'), ('urd','Urdu'), ('uzb','Uzbek'), ('vie','Vietnamese'), ('vol','Volap\xc3\xbck'), ('wol','Wolof'), ('xho','Xhosa'), ('yid','Yiddish'), ('yor','Yoruba'), ('zha','Zhuang'), ('zul','Zulu')], default='eng', help_text='language used within the dataset', max_length=3, verbose_name='language'), + ), + ] diff --git a/geonode/base/migrations/0037_auto_20200305_1520.py b/geonode/base/migrations/0037_auto_20200305_1520.py new file mode 100644 index 0000000..01f341e --- /dev/null +++ b/geonode/base/migrations/0037_auto_20200305_1520.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.10 on 2020-03-05 15:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0036_resourcebase_doi'), + ] + + operations = [ + migrations.AlterField( + model_name='curatedthumbnail', + name='resource', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase'), + ), + migrations.AlterField( + model_name='resourcebase', + name='created', + field=models.DateTimeField(auto_now_add=True, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='last_updated', + field=models.DateTimeField(auto_now=True, null=True), + ), + migrations.AlterField( + model_name='resourcebase', + name='regions', + field=models.ManyToManyField(blank=True, help_text='keyword identifies a location', null=True, to='base.Region', verbose_name='keywords region'), + ), + migrations.AlterField( + model_name='resourcebase', + name='tkeywords', + field=models.ManyToManyField(blank=True, help_text='formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject (space or comma-separated', null=True, to='base.ThesaurusKeyword', verbose_name='keywords'), + ), + ] diff --git a/geonode/base/migrations/0038_auto_20200318_0953.py b/geonode/base/migrations/0038_auto_20200318_0953.py new file mode 100644 index 0000000..0684ea3 --- /dev/null +++ b/geonode/base/migrations/0038_auto_20200318_0953.py @@ -0,0 +1,45 @@ +# Generated by Django 2.2.11 on 2020-03-18 09:53 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('groups', '0032_merge_20200306_1153'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('base', '0037_auto_20200305_1520'), + ] + + operations = [ + migrations.CreateModel( + name='UserGeoLimit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('wkt', models.TextField(blank=True, db_column='wkt')), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='GroupGeoLimit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('wkt', models.TextField(blank=True, db_column='wkt')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='groups.GroupProfile')), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase')), + ], + ), + migrations.AddField( + model_name='resourcebase', + name='groups_geolimits', + field=models.ManyToManyField(blank=True, null=True, related_name='groups_geolimits', to='base.GroupGeoLimit'), + ), + migrations.AddField( + model_name='resourcebase', + name='users_geolimits', + field=models.ManyToManyField(blank=True, null=True, related_name='users_geolimits', to='base.UserGeoLimit'), + ), + ] diff --git a/geonode/base/migrations/0038_configuration.py b/geonode/base/migrations/0038_configuration.py new file mode 100644 index 0000000..da117c9 --- /dev/null +++ b/geonode/base/migrations/0038_configuration.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.11 on 2020-03-09 08:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0037_auto_20200305_1520'), + ] + + operations = [ + migrations.CreateModel( + name='Configuration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('read_only', models.BooleanField(default=False)), + ('maintenance', models.BooleanField(default=False)), + ], + options={ + 'verbose_name_plural': 'Configuration', + }, + ), + ] diff --git a/geonode/base/migrations/0038_delete_backup.py b/geonode/base/migrations/0038_delete_backup.py new file mode 100644 index 0000000..e8faf92 --- /dev/null +++ b/geonode/base/migrations/0038_delete_backup.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.11 on 2020-03-20 11:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0037_auto_20200305_1520'), + ] + + operations = [ + migrations.DeleteModel( + name='Backup', + ), + ] diff --git a/geonode/base/migrations/0039_auto_20200321_1338.py b/geonode/base/migrations/0039_auto_20200321_1338.py new file mode 100644 index 0000000..cb1cd39 --- /dev/null +++ b/geonode/base/migrations/0039_auto_20200321_1338.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.11 on 2020-03-21 13:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0038_configuration'), + ] + + operations = [ + migrations.AlterField( + model_name='license', + name='name', + field=models.CharField(max_length=255), + ), + migrations.AlterField( + model_name='license', + name='name_en', + field=models.CharField(max_length=255, null=True), + ), + ] diff --git a/geonode/base/migrations/0040_merge_20200321_2245.py b/geonode/base/migrations/0040_merge_20200321_2245.py new file mode 100644 index 0000000..c3b4aaa --- /dev/null +++ b/geonode/base/migrations/0040_merge_20200321_2245.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.11 on 2020-03-21 22:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0038_delete_backup'), + ('base', '0039_auto_20200321_1338'), + ] + + operations = [ + ] diff --git a/geonode/base/migrations/0041_merge_20200323_1119.py b/geonode/base/migrations/0041_merge_20200323_1119.py new file mode 100644 index 0000000..e8fcc8a --- /dev/null +++ b/geonode/base/migrations/0041_merge_20200323_1119.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.11 on 2020-03-23 11:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0038_auto_20200318_0953'), + ('base', '0040_merge_20200321_2245'), + ] + + operations = [ + ] diff --git a/geonode/base/migrations/0042_auto_20200414_1554.py b/geonode/base/migrations/0042_auto_20200414_1554.py new file mode 100644 index 0000000..53e2307 --- /dev/null +++ b/geonode/base/migrations/0042_auto_20200414_1554.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.12 on 2020-04-14 15:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0041_merge_20200323_1119'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='doi', + field=models.CharField(blank=True, help_text='a DOI will be added by Admin before publication.', max_length=255, null=True, verbose_name='DOI'), + ), + ] diff --git a/geonode/base/migrations/0043_auto_20200527_0833.py b/geonode/base/migrations/0043_auto_20200527_0833.py new file mode 100644 index 0000000..dc48ca2 --- /dev/null +++ b/geonode/base/migrations/0043_auto_20200527_0833.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.12 on 2020-05-27 08:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0042_auto_20200414_1554'), + ] + + operations = [ + migrations.AlterField( + model_name='hierarchicalkeyword', + name='name', + field=models.CharField(max_length=100, unique=True, verbose_name='name'), + ), + migrations.AlterField( + model_name='hierarchicalkeyword', + name='slug', + field=models.SlugField(max_length=100, unique=True, verbose_name='slug'), + ), + ] diff --git a/geonode/base/migrations/0044_auto_20201001_1514.py b/geonode/base/migrations/0044_auto_20201001_1514.py new file mode 100644 index 0000000..6db661b --- /dev/null +++ b/geonode/base/migrations/0044_auto_20201001_1514.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.15 on 2020-10-01 15:14 + +from django.db import migrations, models +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0043_auto_20200527_0833'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='keywords', + field=taggit.managers.TaggableManager(blank=True, help_text='commonly used word(s) or formalised word(s) or phrase(s) used to describe the subject (space or comma-separated)', through='base.TaggedContentItem', to='base.HierarchicalKeyword', verbose_name='keywords'), + ), + migrations.AlterField( + model_name='resourcebase', + name='tkeywords', + field=models.ManyToManyField(blank=True, help_text='formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject (space or comma-separated)', null=True, to='base.ThesaurusKeyword', verbose_name='keywords'), + ), + ] diff --git a/geonode/base/migrations/0044_resourcebase_bbox_polygon.py b/geonode/base/migrations/0044_resourcebase_bbox_polygon.py new file mode 100644 index 0000000..28252a3 --- /dev/null +++ b/geonode/base/migrations/0044_resourcebase_bbox_polygon.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.11 on 2020-03-23 19:44 + +import django.contrib.gis.db.models.fields +from django.db import migrations +from django.contrib.gis.geos import Polygon + + +def populate_polygon(apps, schema_editor): + ResourceBase = apps.get_model('base', 'ResourceBase') + for res in ResourceBase.objects.all(): + bbox = [getattr(res, key, None) for key in ('bbox_x0', 'bbox_y0', 'bbox_x1', 'bbox_y1')] + if all(bbox): + res.bbox_polygon = Polygon.from_bbox(bbox) + res.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0043_auto_20200527_0833'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='bbox_polygon', + field=django.contrib.gis.db.models.fields.PolygonField(null=True, blank=True, srid=4326), + ), + migrations.RunPython(populate_polygon, migrations.RunPython.noop), + ] diff --git a/geonode/base/migrations/0045_auto_20200507_0445.py b/geonode/base/migrations/0045_auto_20200507_0445.py new file mode 100644 index 0000000..7f639e3 --- /dev/null +++ b/geonode/base/migrations/0045_auto_20200507_0445.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.12 on 2020-05-07 04:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0044_resourcebase_bbox_polygon'), + ('layers', '0028_auto_20200323_1945'), + ] + + operations = [ + migrations.RemoveField( + model_name='resourcebase', + name='bbox_x0', + ), + migrations.RemoveField( + model_name='resourcebase', + name='bbox_x1', + ), + migrations.RemoveField( + model_name='resourcebase', + name='bbox_y0', + ), + migrations.RemoveField( + model_name='resourcebase', + name='bbox_y1', + ), + ] diff --git a/geonode/base/migrations/0046_merge_20201016_0937.py b/geonode/base/migrations/0046_merge_20201016_0937.py new file mode 100644 index 0000000..7071fb5 --- /dev/null +++ b/geonode/base/migrations/0046_merge_20201016_0937.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.15 on 2020-10-16 09:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0045_auto_20200507_0445'), + ('base', '0044_auto_20201001_1514'), + ] + + operations = [ + ] diff --git a/geonode/base/migrations/0047_resourcebase_attribution.py b/geonode/base/migrations/0047_resourcebase_attribution.py new file mode 100644 index 0000000..926c50f --- /dev/null +++ b/geonode/base/migrations/0047_resourcebase_attribution.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2020-11-05 22:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0046_merge_20201016_0937'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='attribution', + field=models.CharField(blank=True, help_text='authority or function assigned, as to a ruler, legislative assembly, delegate, or the like.', max_length=2048, null=True, verbose_name='Attribution'), + ), + ] diff --git a/geonode/base/migrations/0048_auto_20201116_0914.py b/geonode/base/migrations/0048_auto_20201116_0914.py new file mode 100644 index 0000000..d3686d0 --- /dev/null +++ b/geonode/base/migrations/0048_auto_20201116_0914.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.16 on 2020-11-16 09:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0047_resourcebase_attribution'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='category', + field=models.ForeignKey(blank=True, help_text='high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets.', limit_choices_to=models.Q(is_choice=True), null=True, on_delete=django.db.models.deletion.SET_NULL, to='base.TopicCategory'), + ), + migrations.AlterField( + model_name='resourcebase', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='auth.Group'), + ), + migrations.AlterField( + model_name='resourcebase', + name='license', + field=models.ForeignKey(blank=True, help_text='license of the dataset', null=True, on_delete=django.db.models.deletion.SET_NULL, to='base.License', verbose_name='License'), + ), + migrations.AlterField( + model_name='resourcebase', + name='restriction_code_type', + field=models.ForeignKey(blank=True, help_text='limitation(s) placed upon the access or use of the data.', limit_choices_to=models.Q(is_choice=True), null=True, on_delete=django.db.models.deletion.SET_NULL, to='base.RestrictionCodeType', verbose_name='restrictions'), + ), + migrations.AlterField( + model_name='resourcebase', + name='spatial_representation_type', + field=models.ForeignKey(blank=True, help_text='method used to represent geographic information in the dataset.', limit_choices_to=models.Q(is_choice=True), null=True, on_delete=django.db.models.deletion.SET_NULL, to='base.SpatialRepresentationType', verbose_name='spatial representation type'), + ), + ] diff --git a/geonode/base/migrations/0049_resourcebase_resource_type.py b/geonode/base/migrations/0049_resourcebase_resource_type.py new file mode 100644 index 0000000..dce0c1d --- /dev/null +++ b/geonode/base/migrations/0049_resourcebase_resource_type.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2020-12-02 14:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0048_auto_20201116_0914'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='resource_type', + field=models.CharField(blank=True, max_length=1024, null=True, verbose_name='Resource Type'), + ), + ] diff --git a/geonode/base/migrations/0050_auto_20210201_1008.py b/geonode/base/migrations/0050_auto_20210201_1008.py new file mode 100644 index 0000000..f51b594 --- /dev/null +++ b/geonode/base/migrations/0050_auto_20210201_1008.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.16 on 2021-02-01 10:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0049_resourcebase_resource_type'), + ] + + operations = [ + migrations.AddField( + model_name='thesaurus', + name='about', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.CreateModel( + name='ThesaurusLabel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('lang', models.CharField(max_length=3)), + ('label', models.CharField(max_length=255)), + ('thesaurus', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rel_thesaurus', to='base.Thesaurus')), + ], + options={ + 'verbose_name_plural': 'Thesaurus Labels', + 'ordering': ('lang',), + 'unique_together': {('thesaurus', 'lang')}, + }, + ), + ] diff --git a/geonode/base/migrations/0051_auto_20210202_1656.py b/geonode/base/migrations/0051_auto_20210202_1656.py new file mode 100644 index 0000000..711e15e --- /dev/null +++ b/geonode/base/migrations/0051_auto_20210202_1656.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.16 on 2021-02-02 16:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0050_auto_20210201_1008'), + ] + + operations = [ + migrations.AddField( + model_name='thesaurus', + name='card_max', + field=models.IntegerField(default=True), + ), + migrations.AddField( + model_name='thesaurus', + name='card_min', + field=models.IntegerField(default=True), + ), + migrations.AddField( + model_name='thesaurus', + name='facet', + field=models.BooleanField(default=True), + ), + ] diff --git a/geonode/base/migrations/0052_auto_20210205_1631.py b/geonode/base/migrations/0052_auto_20210205_1631.py new file mode 100644 index 0000000..2b7c17e --- /dev/null +++ b/geonode/base/migrations/0052_auto_20210205_1631.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.16 on 2021-02-05 16:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0051_auto_20210202_1656'), + ] + + operations = [ + migrations.AlterField( + model_name='thesaurus', + name='card_max', + field=models.IntegerField(default=-1), + ), + migrations.AlterField( + model_name='thesaurus', + name='card_min', + field=models.IntegerField(default=0), + ), + ] diff --git a/geonode/base/migrations/0053_auto_20210223_0905.py b/geonode/base/migrations/0053_auto_20210223_0905.py new file mode 100644 index 0000000..ba5603e --- /dev/null +++ b/geonode/base/migrations/0053_auto_20210223_0905.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-02-23 09:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0052_auto_20210205_1631'), + ] + + operations = [ + migrations.AlterField( + model_name='thesaurus', + name='id', + field=models.PositiveIntegerField(primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/geonode/base/migrations/0054_auto_20210223_1202.py b/geonode/base/migrations/0054_auto_20210223_1202.py new file mode 100644 index 0000000..b25e5c0 --- /dev/null +++ b/geonode/base/migrations/0054_auto_20210223_1202.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-02-23 12:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0053_auto_20210223_0905'), + ] + + operations = [ + migrations.AlterField( + model_name='thesaurus', + name='id', + field=models.AutoField(primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/geonode/base/migrations/0055_auto_20210302_1109.py b/geonode/base/migrations/0055_auto_20210302_1109.py new file mode 100644 index 0000000..004f99c --- /dev/null +++ b/geonode/base/migrations/0055_auto_20210302_1109.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.16 on 2021-03-02 11:09 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0054_auto_20210223_1202'), + ] + + operations = [ + migrations.AlterModelOptions( + name='thesauruskeywordlabel', + options={'ordering': ('keyword', 'lang'), 'verbose_name_plural': 'Thesaurus Keyword Labels'}, + ), + ] diff --git a/geonode/base/migrations/0056_resourcebase_ll_bbox_polygon.py b/geonode/base/migrations/0056_resourcebase_ll_bbox_polygon.py new file mode 100644 index 0000000..20e8f04 --- /dev/null +++ b/geonode/base/migrations/0056_resourcebase_ll_bbox_polygon.py @@ -0,0 +1,47 @@ +# Generated by Django 2.2.16 on 2021-03-02 11:41 +import logging + +import django.contrib.gis.db.models.fields +from django.contrib.gis.geos import Polygon + +from django.db import migrations + +logger = logging.getLogger(__name__) + + +def populate_ll_polygon(apps, schema_editor): + ResourceBase = apps.get_model('base', 'ResourceBase') + for res in ResourceBase.objects.all(): + try: + bbox = res.bbox_polygon + srid = None + srid = int(res.srid.split(':')[1]) if res.srid else 4326 + # pre-catch well known data problems + if srid == 404000: + logger.warning(f"Skipping resource ID:{res.id} '{res.title}' SRID:{res.srid} ({srid})") + continue + + if bbox and srid: + bbox = bbox.clone() + bbox.srid = srid + bbox.transform(4326) + res.ll_bbox_polygon = bbox + res.save() + except Exception as e: + logger.warning(f"Error while transforming resource ID:{res.id} '{res.title}' SRID:{res.srid} ({srid}): {e}", exc_info=e) + raise e + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0055_auto_20210302_1109'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='ll_bbox_polygon', + field=django.contrib.gis.db.models.fields.PolygonField(blank=True, null=True, srid=4326), + ), + migrations.RunPython(populate_ll_polygon, migrations.RunPython.noop), + ] diff --git a/geonode/base/migrations/0057_resourcebase_metadata_only.py b/geonode/base/migrations/0057_resourcebase_metadata_only.py new file mode 100644 index 0000000..47d99d1 --- /dev/null +++ b/geonode/base/migrations/0057_resourcebase_metadata_only.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-03-10 11:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0056_resourcebase_ll_bbox_polygon'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='metadata_only', + field=models.BooleanField(default=False, help_text='If true, will be excluded from search', verbose_name='Metadata'), + ), + ] diff --git a/geonode/base/migrations/0058_thesaurus_order.py b/geonode/base/migrations/0058_thesaurus_order.py new file mode 100644 index 0000000..20ea856 --- /dev/null +++ b/geonode/base/migrations/0058_thesaurus_order.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-04-06 07:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0057_resourcebase_metadata_only'), + ] + + operations = [ + migrations.AddField( + model_name='thesaurus', + name='order', + field=models.IntegerField(default=0), + ), + ] diff --git a/geonode/base/migrations/0059_fill_empty_resource_type.py b/geonode/base/migrations/0059_fill_empty_resource_type.py new file mode 100644 index 0000000..e46e35d --- /dev/null +++ b/geonode/base/migrations/0059_fill_empty_resource_type.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.16 on 2021-05-03 09:40 + +from django.db import migrations +from django.contrib.contenttypes.models import ContentType +from django.db.models import Q +import logging +logger = logging.getLogger(__name__) + +def set_resource_type(apps, _): + try: + MyModel = apps.get_model('base', 'ResourceBase') + resources = MyModel.objects.filter(Q(resource_type=None) | Q(resource_type="")) + for resource in resources: + rtype = ContentType.objects.get(id=resource.polymorphic_ctype_id) + resource.resource_type=rtype.model + resource.save(update_fields=['resource_type']) + except Exception as e: + logger.error(f'Error during set resource_type: {e.args[0]}') + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0058_thesaurus_order'), + ] + + operations = [ + migrations.RunPython(set_resource_type, migrations.RunPython.noop), + ] diff --git a/geonode/base/migrations/0060_auto_20210512_1658.py b/geonode/base/migrations/0060_auto_20210512_1658.py new file mode 100644 index 0000000..6a322fe --- /dev/null +++ b/geonode/base/migrations/0060_auto_20210512_1658.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.16 on 2021-05-12 16:58 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0059_fill_empty_resource_type'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='owned_resource', to=settings.AUTH_USER_MODEL, verbose_name='Owner'), + ), + ] diff --git a/geonode/base/migrations/0060_resourcebase_state.py b/geonode/base/migrations/0060_resourcebase_state.py new file mode 100644 index 0000000..6711354 --- /dev/null +++ b/geonode/base/migrations/0060_resourcebase_state.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.20 on 2021-05-25 09:29 + +from django.db import migrations, models +from geonode.base import enumerations + + +def update_state_value(apps, schema_editor): + """Let's update the existing resources to the 'PROCESSED' state + otherwise those will be set to 'dirty' later on and hidden by the APIs. + """ + MyModel = apps.get_model('base', 'ResourceBase') + MyModel.objects.filter().update(state=enumerations.STATE_PROCESSED) + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0064_resourcebase_metadata'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='state', + field=models.CharField(choices=[('READY', 'READY'), ('RUNNING', 'RUNNING'), ('PENDING', 'PENDING'), ('WAITING', 'WAITING'), ('INCOMPLETE', 'INCOMPLETE'), ('COMPLETE', 'COMPLETE'), ('INVALID', 'INVALID'), ('PROCESSED', 'PROCESSED')], default='READY', help_text='Hold the resource processing state.', max_length=16, verbose_name='State'), + ), + migrations.RunPython(update_state_value, migrations.RunPython.noop) + ] diff --git a/geonode/base/migrations/0061_auto_20211117_1238.py b/geonode/base/migrations/0061_auto_20211117_1238.py new file mode 100644 index 0000000..c131b90 --- /dev/null +++ b/geonode/base/migrations/0061_auto_20211117_1238.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.7 on 2021-11-19 13:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0060_auto_20210512_1658'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='was_approved', + field=models.BooleanField(default=True, help_text='Previous Approved state.', verbose_name='Was Approved'), + ), + migrations.AddField( + model_name='resourcebase', + name='was_published', + field=models.BooleanField(default=True, help_text='Previous Published state.', verbose_name='Was Published'), + ), + ] diff --git a/geonode/base/migrations/0061_merge_0060_auto_20210512_1658_0060_resourcebase_state.py b/geonode/base/migrations/0061_merge_0060_auto_20210512_1658_0060_resourcebase_state.py new file mode 100644 index 0000000..fb82801 --- /dev/null +++ b/geonode/base/migrations/0061_merge_0060_auto_20210512_1658_0060_resourcebase_state.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2 on 2021-05-25 15:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0061_auto_20211117_1238'), + ('base', '0060_resourcebase_state'), + ] + + operations = [ + ] diff --git a/geonode/base/migrations/0062_resourcebase_extra_metadata.py b/geonode/base/migrations/0062_resourcebase_extra_metadata.py new file mode 100644 index 0000000..25b46f2 --- /dev/null +++ b/geonode/base/migrations/0062_resourcebase_extra_metadata.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.24 on 2022-01-26 14:04 + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0061_auto_20211117_1238'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='extra_metadata', + field=models.JSONField(blank=True, default=list, null=True), + ), + ] \ No newline at end of file diff --git a/geonode/base/migrations/0062_resourcebase_files.py b/geonode/base/migrations/0062_resourcebase_files.py new file mode 100644 index 0000000..07db217 --- /dev/null +++ b/geonode/base/migrations/0062_resourcebase_files.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2021-05-28 15:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0061_merge_0060_auto_20210512_1658_0060_resourcebase_state'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='files', + field=models.JSONField(default=dict), + ), + ] diff --git a/geonode/base/migrations/0063_alter_resourcebase_files.py b/geonode/base/migrations/0063_alter_resourcebase_files.py new file mode 100644 index 0000000..ff0dfae --- /dev/null +++ b/geonode/base/migrations/0063_alter_resourcebase_files.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2021-06-03 16:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0062_resourcebase_files'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='files', + field=models.JSONField(default=dict, null=True), + ), + ] diff --git a/geonode/base/migrations/0063_auto_20220128_1042.py b/geonode/base/migrations/0063_auto_20220128_1042.py new file mode 100644 index 0000000..4051993 --- /dev/null +++ b/geonode/base/migrations/0063_auto_20220128_1042.py @@ -0,0 +1,26 @@ +# Generated by Django 2.2.24 on 2022-01-28 10:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0062_resourcebase_extra_metadata'), + ] + + operations = [ + migrations.RemoveField( + model_name='resourcebase', + name='extra_metadata', + ), + migrations.CreateModel( + name='ExtraMetadata', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('metadata', models.JSONField(blank=True, default=dict, null=True)), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase')), + ], + ), + ] \ No newline at end of file diff --git a/geonode/base/migrations/0064_alter_resourcebase_files.py b/geonode/base/migrations/0064_alter_resourcebase_files.py new file mode 100644 index 0000000..c9af714 --- /dev/null +++ b/geonode/base/migrations/0064_alter_resourcebase_files.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2 on 2021-06-04 07:42 + +from django.db import migrations, models +from geonode.base.models import ResourceBase + +def change_value(apps, schema_editor): + resources = apps.get_model('base', 'ResourceBase') + for r in resources.objects.all(): + if isinstance(r.files, dict): + r2 = ResourceBase.objects.filter(id=r.id) + f = list(r.files.values()) + r2.update(**{'files': f}) + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0063_alter_resourcebase_files'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='files', + field=models.JSONField(blank=True, default=list, null=True), + ), + migrations.RunPython(change_value, migrations.RunPython.noop), + + ] diff --git a/geonode/base/migrations/0064_resourcebase_metadata.py b/geonode/base/migrations/0064_resourcebase_metadata.py new file mode 100644 index 0000000..51634ae --- /dev/null +++ b/geonode/base/migrations/0064_resourcebase_metadata.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.24 on 2022-01-28 10:51 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0063_auto_20220128_1042'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='metadata', + field=models.ManyToManyField(blank=True, null=True, to='base.ExtraMetadata', verbose_name='Extra Metadata'), + ), + migrations.AlterField( + model_name='resourcebase', + name='uuid', + field=models.CharField(max_length=36, unique=True, default=str(uuid.uuid4)), + ), + ] \ No newline at end of file diff --git a/geonode/base/migrations/0065_alter_curatedthumbnail_img.py b/geonode/base/migrations/0065_alter_curatedthumbnail_img.py new file mode 100644 index 0000000..9426304 --- /dev/null +++ b/geonode/base/migrations/0065_alter_curatedthumbnail_img.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-06-09 13:58 + +from django.db import migrations, models +import geonode.storage.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0064_alter_resourcebase_files'), + ] + + operations = [ + migrations.AlterField( + model_name='curatedthumbnail', + name='img', + field=models.ImageField(storage=geonode.storage.manager.StorageManager(), upload_to='curated_thumbs'), + ), + ] diff --git a/geonode/base/migrations/0066_resourcebase_data.py b/geonode/base/migrations/0066_resourcebase_data.py new file mode 100644 index 0000000..87a6989 --- /dev/null +++ b/geonode/base/migrations/0066_resourcebase_data.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2021-06-11 13:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0065_alter_curatedthumbnail_img'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='blob', + field=models.JSONField(blank=True, default=dict, null=True), + ), + ] diff --git a/geonode/base/migrations/0067_resourcebase_storetype.py b/geonode/base/migrations/0067_resourcebase_storetype.py new file mode 100644 index 0000000..e280d13 --- /dev/null +++ b/geonode/base/migrations/0067_resourcebase_storetype.py @@ -0,0 +1,52 @@ +# Generated by Django 3.2.4 on 2021-07-06 08:48 + +from django.db import migrations, models +from django.db.migrations import RunSQL, RunPython + +LAYERS_SUBTYPES = { + "dataStore": "vector", + "coverageStore": "raster", + "remoteStore": "remote", + "vectorTimeSeries": "vector_time" +} + +clone_layers_storetypes = ''' +UPDATE base_resourcebase +SET "storetype"=subquery."storeType" +FROM (select resourcebase_ptr_id,"storeType" from layers_layer gg) AS subquery +WHERE base_resourcebase.id=subquery.resourcebase_ptr_id; +''' + +clone_documents_storetypes = ''' +UPDATE base_resourcebase +SET "storetype"=subquery."doc_type" +FROM (select resourcebase_ptr_id,"doc_type" from documents_document gg) AS subquery +WHERE base_resourcebase.id=subquery.resourcebase_ptr_id; +''' + + +def update_storetype_value(apps, schema_editor): + MyModel = apps.get_model('layers', 'Layer') + for l in MyModel.objects.filter(storetype__in=(LAYERS_SUBTYPES.keys())): + u = MyModel.objects.filter(resourcebase_ptr_id=l.resourcebase_ptr_id) + store_type = (lambda element: LAYERS_SUBTYPES.get(element, element))(l.storetype) + u.update(storetype=store_type) + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0066_resourcebase_data'), + ('documents', '24_initial'), # make sure documents_document exists + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='storetype', + field=models.CharField(blank=True, max_length=128, null=True), + ), + RunSQL(clone_layers_storetypes), + RunSQL(clone_documents_storetypes), + RunPython(update_storetype_value, migrations.RunPython.noop) + ] diff --git a/geonode/base/migrations/0068_rename_storetype_resourcebase_subtype.py b/geonode/base/migrations/0068_rename_storetype_resourcebase_subtype.py new file mode 100644 index 0000000..41d0dda --- /dev/null +++ b/geonode/base/migrations/0068_rename_storetype_resourcebase_subtype.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-07-09 10:05 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0067_resourcebase_storetype'), + ] + + operations = [ + migrations.RenameField( + model_name='resourcebase', + old_name='storetype', + new_name='subtype', + ), + ] diff --git a/geonode/base/migrations/0069_resourcebase_sourcetype.py b/geonode/base/migrations/0069_resourcebase_sourcetype.py new file mode 100644 index 0000000..3ed95a5 --- /dev/null +++ b/geonode/base/migrations/0069_resourcebase_sourcetype.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-07-20 15:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0068_rename_storetype_resourcebase_subtype'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='sourcetype', + field=models.CharField(choices=[('LOCAL', 'LOCAL'), ('REMOTE', 'REMOTE'), ('COPYREMOTE', 'COPYREMOTE')], default='LOCAL', help_text='The resource source type, which can be one of "LOCAL", "REMOTE" or "COPYREMOTE".', max_length=16, verbose_name='Source Type'), + ) + ] diff --git a/geonode/base/migrations/0070_auto_20210720_1709.py b/geonode/base/migrations/0070_auto_20210720_1709.py new file mode 100644 index 0000000..95c11fe --- /dev/null +++ b/geonode/base/migrations/0070_auto_20210720_1709.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.4 on 2021-07-20 17:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0069_resourcebase_sourcetype'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='ows_url', + field=models.URLField(blank=True, default='http://localhost:8080/geoserver/ows', help_text='The URL of the OWS service providing this layer, if any exists.', null=True, verbose_name='ows URL'), + ), + migrations.AddField( + model_name='resourcebase', + name='remote_typename', + field=models.CharField(blank=True, help_text='Name of the Remote Service if any.', max_length=512, null=True, verbose_name='Remote Service Typename'), + ) + ] diff --git a/geonode/base/migrations/0071_remove_resourcebase_ows_url.py b/geonode/base/migrations/0071_remove_resourcebase_ows_url.py new file mode 100644 index 0000000..99cdfe3 --- /dev/null +++ b/geonode/base/migrations/0071_remove_resourcebase_ows_url.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-07-21 15:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0070_auto_20210720_1709'), + ] + + operations = [ + migrations.RemoveField( + model_name='resourcebase', + name='ows_url', + ), + ] diff --git a/geonode/base/migrations/0072_remove_resourcebase_detail_url.py b/geonode/base/migrations/0072_remove_resourcebase_detail_url.py new file mode 100644 index 0000000..95ecc95 --- /dev/null +++ b/geonode/base/migrations/0072_remove_resourcebase_detail_url.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-07-22 10:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0071_remove_resourcebase_ows_url'), + ] + + operations = [ + migrations.RemoveField( + model_name='resourcebase', + name='detail_url', + ), + ] diff --git a/geonode/base/migrations/0073_resourcebase_thumbnail_path.py b/geonode/base/migrations/0073_resourcebase_thumbnail_path.py new file mode 100644 index 0000000..4953d04 --- /dev/null +++ b/geonode/base/migrations/0073_resourcebase_thumbnail_path.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-11-12 14:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0072_remove_resourcebase_detail_url'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='thumbnail_path', + field=models.TextField(blank=True, null=True, verbose_name='Thumbnail path'), + ), + ] diff --git a/geonode/base/migrations/0074_drop_curated_thumbs.py b/geonode/base/migrations/0074_drop_curated_thumbs.py new file mode 100644 index 0000000..086a31a --- /dev/null +++ b/geonode/base/migrations/0074_drop_curated_thumbs.py @@ -0,0 +1,46 @@ +from django.db import migrations, connection + +from geonode.storage.manager import storage_manager +from geonode.thumbs.thumbnails import _generate_thumbnail_name +import logging +logger = logging.getLogger(__name__) + + +def update_thumbnail_urls_and_delete_curated_thumbs_folder(apps, schema_editor): + query = ''' + SELECT base_resourcebase.id, base_curatedthumbnail.img FROM base_resourcebase + INNER JOIN base_curatedthumbnail + ON base_resourcebase.id=base_curatedthumbnail.resource_id; + ''' + # use historical model + ResourceBase = apps.get_model('base', 'ResourceBase') + + c = connection.cursor() + c.execute(query) + results = c.fetchall() + for result in results: + resource_id, image = result + resource = ResourceBase.objects.filter(id=resource_id).first() + try: + bytes_file = storage_manager.open(image).read() + except Exception: + bytes_file = None + + if resource and bytes_file: + try: + filename = _generate_thumbnail_name(resource.get_real_instance()) + resource.save_thumbnail(filename, bytes_file) + except Exception as e: + logger.error(f'Error during updating resource: {e.args[0]}', exc_info=e) + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0073_resourcebase_thumbnail_path'), + ('layers', '0044_alter_dataset_unique_together'), + ] + + operations = [ + migrations.RunPython(update_thumbnail_urls_and_delete_curated_thumbs_folder, migrations.RunPython.noop) + ] diff --git a/geonode/base/migrations/0075_delete_curatedthumbnail.py b/geonode/base/migrations/0075_delete_curatedthumbnail.py new file mode 100644 index 0000000..3d52d73 --- /dev/null +++ b/geonode/base/migrations/0075_delete_curatedthumbnail.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.7 on 2022-01-17 14:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0074_drop_curated_thumbs'), + ] + + operations = [ + migrations.DeleteModel( + name='CuratedThumbnail', + ), + ] diff --git a/geonode/base/migrations/0076_auto_20220128_0919.py b/geonode/base/migrations/0076_auto_20220128_0919.py new file mode 100644 index 0000000..7fb0d35 --- /dev/null +++ b/geonode/base/migrations/0076_auto_20220128_0919.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.11 on 2022-01-28 09:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0075_delete_curatedthumbnail'), + ] + + operations = [ + migrations.AlterField( + model_name='thesauruskeywordlabel', + name='lang', + field=models.CharField(max_length=10), + ), + migrations.AlterField( + model_name='thesauruslabel', + name='lang', + field=models.CharField(max_length=10), + ), + ] diff --git a/geonode/base/migrations/0076_merge_20220204_1339.py b/geonode/base/migrations/0076_merge_20220204_1339.py new file mode 100644 index 0000000..99e945f --- /dev/null +++ b/geonode/base/migrations/0076_merge_20220204_1339.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.11 on 2022-02-04 13:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0064_resourcebase_metadata'), + ('base', '0075_delete_curatedthumbnail'), + ] + + operations = [ + ] diff --git a/geonode/base/migrations/0077_merge_20220204_1347.py b/geonode/base/migrations/0077_merge_20220204_1347.py new file mode 100644 index 0000000..98c5aae --- /dev/null +++ b/geonode/base/migrations/0077_merge_20220204_1347.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.12 on 2022-02-04 13:47 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0076_auto_20220128_0919'), + ('base', '0076_merge_20220204_1339'), + ] + + operations = [ + ] diff --git a/geonode/base/migrations/0078_alter_resourcebase_metadata.py b/geonode/base/migrations/0078_alter_resourcebase_metadata.py new file mode 100644 index 0000000..ba56b8f --- /dev/null +++ b/geonode/base/migrations/0078_alter_resourcebase_metadata.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-02-10 14:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0077_merge_20220204_1347'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='metadata', + field=models.ManyToManyField(blank=True, help_text='Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]', null=True, to='base.ExtraMetadata', verbose_name='Extra Metadata'), + ), + ] diff --git a/geonode/base/migrations/0079_alter_resourcebase_alternate.py b/geonode/base/migrations/0079_alter_resourcebase_alternate.py new file mode 100644 index 0000000..faf560c --- /dev/null +++ b/geonode/base/migrations/0079_alter_resourcebase_alternate.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-02-25 16:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0078_alter_resourcebase_metadata'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='alternate', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/geonode/base/migrations/0080_alter_resourcebase_uuid.py b/geonode/base/migrations/0080_alter_resourcebase_uuid.py new file mode 100644 index 0000000..5f9c2d5 --- /dev/null +++ b/geonode/base/migrations/0080_alter_resourcebase_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.12 on 2022-03-28 10:17 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0079_alter_resourcebase_alternate'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='uuid', + field=models.CharField(max_length=36, unique=True, default=str(uuid.uuid4)), + ), + ] diff --git a/geonode/base/migrations/0081_alter_resourcebase_alternate.py b/geonode/base/migrations/0081_alter_resourcebase_alternate.py new file mode 100644 index 0000000..7c030a1 --- /dev/null +++ b/geonode/base/migrations/0081_alter_resourcebase_alternate.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-04-11 22:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0080_alter_resourcebase_uuid'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='alternate', + field=models.CharField(blank=True, max_length=255, null=True, verbose_name='alternate'), + ), + ] diff --git a/geonode/base/migrations/0082_remove_dialogos_comment.py b/geonode/base/migrations/0082_remove_dialogos_comment.py new file mode 100644 index 0000000..98f184e --- /dev/null +++ b/geonode/base/migrations/0082_remove_dialogos_comment.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.12 on 2022-04-11 22:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0081_alter_resourcebase_alternate'), + ] + + operations = [ + migrations.RunSQL("DROP TABLE IF EXISTS dialogos_comment CASCADE;"), + ] diff --git a/geonode/base/migrations/0083_clean_resources_with_missing_thumb.py b/geonode/base/migrations/0083_clean_resources_with_missing_thumb.py new file mode 100644 index 0000000..645c569 --- /dev/null +++ b/geonode/base/migrations/0083_clean_resources_with_missing_thumb.py @@ -0,0 +1,32 @@ +import logging + +from django.db import migrations +from django.templatetags.static import static + +from geonode.thumbs.utils import MISSING_THUMB + +logger = logging.getLogger(__name__) + + +def set_null_thumbnail(apps, _): + "Sets thumbnail_url to null for resources with thumbnail_url=missing_thumb" + try: + resource_model = apps.get_model('base', 'ResourceBase') + link_model = apps.get_model('base', 'Link') + # update thumbnail urls + resource_model.objects.filter(thumbnail_url__icontains=static(MISSING_THUMB)).update(thumbnail_url=None) + # Remove thumbnail links + link_model.objects.filter(resource__thumbnail_url__isnull=True, name='Thumbnail').delete() + except Exception as e: + logger.exception(e) + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0082_remove_dialogos_comment'), + ] + + operations = [ + migrations.RunPython(set_null_thumbnail, migrations.RunPython.noop), + ] diff --git a/geonode/base/migrations/0084_remove_comments_from_actions.py b/geonode/base/migrations/0084_remove_comments_from_actions.py new file mode 100644 index 0000000..cb1c0e7 --- /dev/null +++ b/geonode/base/migrations/0084_remove_comments_from_actions.py @@ -0,0 +1,27 @@ +import logging + +from django.db import migrations + +from actstream.models import Action + +logger = logging.getLogger(__name__) + + +def delete_comments_actions(apps, _): + "Removes Action items related to comments" + try: + Action.objects.filter( + action_object_content_type__model='comment').delete() + except Exception as e: + logger.exception(e) + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0083_clean_resources_with_missing_thumb'), + ] + + operations = [ + migrations.RunPython(delete_comments_actions, migrations.RunPython.noop), + ] diff --git a/geonode/base/migrations/0085_alter_resourcebase_uuid.py b/geonode/base/migrations/0085_alter_resourcebase_uuid.py new file mode 100644 index 0000000..b7ef374 --- /dev/null +++ b/geonode/base/migrations/0085_alter_resourcebase_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.16 on 2022-11-08 10:52 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0084_remove_comments_from_actions'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='uuid', + field=models.CharField(default=uuid.uuid4, max_length=36, unique=True), + ), + ] diff --git a/geonode/base/migrations/0086_linkedresource.py b/geonode/base/migrations/0086_linkedresource.py new file mode 100644 index 0000000..e5b5f42 --- /dev/null +++ b/geonode/base/migrations/0086_linkedresource.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.21 on 2023-10-05 14:29 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("base", "0085_alter_resourcebase_uuid"), + ] + + operations = [ + migrations.CreateModel( + name="LinkedResource", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "source", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, related_name="linked_to", to="base.resourcebase" + ), + ), + ( + "target", + models.ForeignKey( + blank=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="linked_by", + to="base.resourcebase", + ), + ), + ("internal", models.BooleanField(default=False)), + ], + ), + ] diff --git a/geonode/base/migrations/0087_thesauruskeyword_icon.py b/geonode/base/migrations/0087_thesauruskeyword_icon.py new file mode 100644 index 0000000..87dd7e4 --- /dev/null +++ b/geonode/base/migrations/0087_thesauruskeyword_icon.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.22 on 2023-10-18 14:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0086_linkedresource'), + ] + + operations = [ + migrations.AddField( + model_name='thesauruskeyword', + name='icon', + field=models.CharField(blank=True, max_length=512, null=True, help_text='It can be a fa-class name or a URL to an image'), + ), + ] diff --git a/geonode/base/migrations/0088_auto_20231019_1244.py b/geonode/base/migrations/0088_auto_20231019_1244.py new file mode 100644 index 0000000..c96c97f --- /dev/null +++ b/geonode/base/migrations/0088_auto_20231019_1244.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.22 on 2023-10-19 12:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0087_thesauruskeyword_icon'), + ] + + operations = [ + migrations.RemoveField( + model_name='thesauruskeyword', + name='icon', + ), + migrations.AddField( + model_name='thesauruskeyword', + name='image', + field=models.CharField(blank=True, help_text='A URL to an image', max_length=512, null=True), + ), + ] diff --git a/geonode/base/migrations/0089_resourcebase_advertised.py b/geonode/base/migrations/0089_resourcebase_advertised.py new file mode 100644 index 0000000..a15fc7a --- /dev/null +++ b/geonode/base/migrations/0089_resourcebase_advertised.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-01-16 14:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0088_auto_20231019_1244'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='advertised', + field=models.BooleanField(default=True, help_text='If False, will hide the resource from search results and catalog listings', verbose_name='Advertised'), + ), + ] diff --git a/geonode/base/migrations/0090_alter_resourcebase_polymorphic_ctype.py b/geonode/base/migrations/0090_alter_resourcebase_polymorphic_ctype.py new file mode 100644 index 0000000..7b79121 --- /dev/null +++ b/geonode/base/migrations/0090_alter_resourcebase_polymorphic_ctype.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.9 on 2024-01-10 14:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("base", "0089_resourcebase_advertised"), + ] + + operations = [ + migrations.AlterField( + model_name="resourcebase", + name="polymorphic_ctype", + field=models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="polymorphic_%(app_label)s.%(class)s_set+", + to="contenttypes.contenttype", + ), + ), + ] diff --git a/geonode/base/migrations/0091_alter_hierarchicalkeyword_slug.py b/geonode/base/migrations/0091_alter_hierarchicalkeyword_slug.py new file mode 100644 index 0000000..e884c56 --- /dev/null +++ b/geonode/base/migrations/0091_alter_hierarchicalkeyword_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.9 on 2024-05-21 08:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0090_alter_resourcebase_polymorphic_ctype"), + ] + + operations = [ + migrations.AlterField( + model_name="hierarchicalkeyword", + name="slug", + field=models.SlugField(allow_unicode=True, max_length=100, unique=True, verbose_name="slug"), + ), + ] diff --git a/geonode/base/migrations/0091_create_link_asset_alter_link_type.py b/geonode/base/migrations/0091_create_link_asset_alter_link_type.py new file mode 100644 index 0000000..e3310ca --- /dev/null +++ b/geonode/base/migrations/0091_create_link_asset_alter_link_type.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.9 on 2024-04-24 10:34 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0091_alter_hierarchicalkeyword_slug"), + ("assets", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="link", + name="asset", + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to="assets.asset"), + ), + migrations.AlterField( + model_name="link", + name="link_type", + field=models.CharField( + choices=[ + ("original", "original"), + ("uploaded", "uploaded"), + ("data", "data"), + ("image", "image"), + ("metadata", "metadata"), + ("html", "html"), + ("OGC:WMS", "OGC:WMS"), + ("OGC:WFS", "OGC:WFS"), + ("OGC:WCS", "OGC:WCS"), + ], + max_length=255, + ), + ), + ] diff --git a/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py b/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py new file mode 100644 index 0000000..76b564c --- /dev/null +++ b/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py @@ -0,0 +1,95 @@ +# Generated by Django 4.2.9 on 2024-03-12 11:55 +import logging +import os + +from django.db import migrations +from django.db.models import Q +from django.conf import settings +from django.contrib.auth import get_user_model +from django.urls import reverse + +from geonode.base.models import Link +from geonode.assets.models import LocalAsset +from geonode.utils import build_absolute_uri + +logger = logging.getLogger(__name__) + + +def migrate_files(apps, schema_editor): + + def get_ext(filename): + try: + return os.path.splitext(filename)[1][1:] + except Exception as e: + logger.warning(f"Could not find extension for Resource '{res_hm.title}, file '{filename}': {e}") + return None + + ResourceBase_hm = apps.get_model('base', 'ResourceBase') + Dataset_hm = apps.get_model('layers', 'Dataset') + Document_hm = apps.get_model('documents', 'Document') + + if hasattr(ResourceBase_hm, "files"): + # looping on available resources with files to generate the LocalAssets + for res_hm in ResourceBase_hm.objects.exclude(Q(files__isnull=True) | Q(files__exact=[])).iterator(): + # resolving the real owner instance, since resource.owner is an historical model and cant be used directly + owner = get_user_model().objects.get(pk=res_hm.owner.id) + # logger.warning(f"Creating ASSET for {resource.id} -- owner:{type(resource.owner)} --> {resource.owner}") + + files = res_hm.files + # creating the local asset object + asset = LocalAsset( + title="Files", + description="Original uploaded files", + owner=owner, + location=files + ) + asset.save() + + ### creating the association between asset and Link + + # no existing "uploaded" links exist, so create them right away + # otherwise we create the link with the assigned asset + if dataset_hm := Dataset_hm.objects.filter(pk=res_hm.id).first(): + url = build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + elif doc_hm := Document_hm.objects.filter(pk=res_hm.id).first(): + url = build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + else: + raise TypeError(f'ResourceBase {res_hm.id}::"{res_hm.title} has unhandled type"') + + if len(files) == 1: + ext = get_ext(files[0]) + else: + ext = None + for file in files: + for filetype in settings.SUPPORTED_DATASET_FILE_TYPES: + file_ext = get_ext(file) + if file_ext in filetype["ext"]: + ext = filetype["id"] + break + if ext: + break + + Link.objects.create( + resource_id=res_hm.id, + asset=asset, + link_type="uploaded", + name="Original upload", + extension=ext or "unknown", + url=url + ) + + +class Migration(migrations.Migration): + + dependencies = [ + + ("base", "0091_create_link_asset_alter_link_type"), + ] + + operations = [ + migrations.RunPython(migrate_files, migrations.RunPython.noop), + migrations.RemoveField( + model_name="resourcebase", + name="files", + ), + ] diff --git a/geonode/base/migrations/0093_alter_thesaurus_slug.py b/geonode/base/migrations/0093_alter_thesaurus_slug.py new file mode 100644 index 0000000..190aa6c --- /dev/null +++ b/geonode/base/migrations/0093_alter_thesaurus_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.20 on 2025-06-03 14:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0092_migrate and_remove_resourcebase_files"), + ] + + operations = [ + migrations.AlterField( + model_name="thesaurus", + name="slug", + field=models.CharField(blank=True, default="", max_length=64, null=True), + ), + ] \ No newline at end of file diff --git a/geonode/base/migrations/0094_fix_otherrestrictions_codetype.py b/geonode/base/migrations/0094_fix_otherrestrictions_codetype.py new file mode 100644 index 0000000..a06aae7 --- /dev/null +++ b/geonode/base/migrations/0094_fix_otherrestrictions_codetype.py @@ -0,0 +1,28 @@ +from django.db import migrations +from geonode.base.models import RestrictionCodeType + +def fix_otherrestrictions_codetype(apps, schema_editor): + """ + Fixes the identifier, description and gn_description for the 'otherRestrictions' RestrictionCodeType. + """ + try: + obj = RestrictionCodeType.objects.get(identifier='limitation not listed') + obj.identifier = 'otherRestrictions' + obj.description = 'limitation not listed' + obj.gn_description = 'limitation not listed' + obj.save() + except RestrictionCodeType.DoesNotExist: + pass + + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0093_alter_thesaurus_slug'), + ] + + operations = [ + migrations.RunPython(fix_otherrestrictions_codetype,migrations.RunPython.noop), + ] + \ No newline at end of file diff --git a/geonode/base/migrations/24_initial.py b/geonode/base/migrations/24_initial.py new file mode 100644 index 0000000..a18839e --- /dev/null +++ b/geonode/base/migrations/24_initial.py @@ -0,0 +1,226 @@ +from django.db import migrations, models +import mptt.fields +import geonode.security.models +from django.conf import settings +from django.utils.timezone import now + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='ContactRole', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('role', models.CharField(help_text='function performed by the responsible party', max_length=255, choices=[('author', 'party who authored the resource'), ('processor', 'party who has processed the data in a manner such that the resource has been modified'), ('publisher', 'party who published the resource'), ('custodian', 'party that accepts accountability and responsibility for the data and ensures appropriate care and maintenance of the resource'), ('pointOfContact', 'party who can be contacted for acquiring knowledge about or acquisition of the resource'), ('distributor', 'party who distributes the resource'), ('user', 'party who uses the resource'), ('resourceProvider', 'party that supplies the resource'), ('originator', 'party who created the resource'), ('owner', 'party that owns the resource'), ('principalInvestigator', 'key party responsible for gathering information and conducting research')])), + ('contact', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), + ], + ), + migrations.CreateModel( + name='License', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('identifier', models.CharField(max_length=255, editable=False)), + ('name', models.CharField(max_length=100)), + ('name_en', models.CharField(max_length=100, null=True)), + ('abbreviation', models.CharField(max_length=20, null=True, blank=True)), + ('description', models.TextField(null=True, blank=True)), + ('description_en', models.TextField(null=True, blank=True)), + ('url', models.URLField(max_length=2000, null=True, blank=True)), + ('license_text', models.TextField(null=True, blank=True)), + ('license_text_en', models.TextField(null=True, blank=True)), + ], + options={ + 'ordering': ('name',), + 'verbose_name_plural': 'Licenses', + }, + ), + migrations.CreateModel( + name='Link', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('extension', models.CharField(help_text='For example "kml"', max_length=255)), + ('link_type', models.CharField(max_length=255, choices=[('original','original'), ('data','data'), ('image','image'), ('metadata','metadata'), ('html','html'), ('OGC:WMS','OGC:WMS'), ('OGC:WFS','OGC:WFS'), ('OGC:WCS','OGC:WCS')])), + ('name', models.CharField(help_text='For example "View in Google Earth"', max_length=255)), + ('mime', models.CharField(help_text='For example "text/xml"', max_length=255)), + ('url', models.TextField(max_length=1000)), + ], + ), + migrations.CreateModel( + name='Region', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('code', models.CharField(unique=True, max_length=50)), + ('name', models.CharField(max_length=255)), + ('name_en', models.CharField(max_length=255, null=True)), + ('lft', models.PositiveIntegerField(editable=False, db_index=True)), + ('rght', models.PositiveIntegerField(editable=False, db_index=True)), + ('tree_id', models.PositiveIntegerField(editable=False, db_index=True)), + ('level', models.PositiveIntegerField(editable=False, db_index=True)), + ('parent', mptt.fields.TreeForeignKey(related_name='children', + on_delete=models.CASCADE, blank=True, to='base.Region', null=True)), + ], + options={ + 'ordering': ('name',), + 'verbose_name_plural': 'Metadata Regions', + }, + ), + migrations.CreateModel( + name='ResourceBase', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('uuid', models.CharField(max_length=36)), + ('title', models.CharField(help_text='name by which the cited resource is known', max_length=255, verbose_name='title')), + ('date', models.DateTimeField(default=now, help_text='reference date for the cited resource', verbose_name='date')), + ('date_type', models.CharField(default='publication', help_text='identification of when a given event occurred', max_length=255, verbose_name='date type', choices=[('creation', 'Creation'), ('publication', 'Publication'), ('revision', 'Revision')])), + ('edition', models.CharField(help_text='version of the cited resource', max_length=255, null=True, verbose_name='edition', blank=True)), + ('abstract', models.TextField(help_text='brief narrative summary of the content of the resource(s)', verbose_name='abstract', blank=True)), + ('purpose', models.TextField(help_text='summary of the intentions with which the resource(s) was developed', null=True, verbose_name='purpose', blank=True)), + ('maintenance_frequency', models.CharField(choices=[('unknown', 'frequency of maintenance for the data is not known'), ('continual', 'data is repeatedly and frequently updated'), ('notPlanned', 'there are no plans to update the data'), ('daily', 'data is updated each day'), ('annually', 'data is updated every year'), ('asNeeded', 'data is updated as deemed necessary'), ('monthly', 'data is updated each month'), ('fortnightly', 'data is updated every two weeks'), ('irregular', 'data is updated in intervals that are uneven in duration'), ('weekly', 'data is updated on a weekly basis'), ('biannually', 'data is updated twice each year'), ('quarterly', 'data is updated every three months')], max_length=255, blank=True, help_text='frequency with which modifications and deletions are made to the data after it is first produced', null=True, verbose_name='maintenance frequency')), + ('constraints_other', models.TextField(help_text='other restrictions and legal prerequisites for accessing and using the resource or metadata', null=True, verbose_name='restrictions other', blank=True)), + ('language', models.CharField(default='eng', help_text='language used within the dataset', max_length=3, verbose_name='language', choices=[('abk','Abkhazian'), ('aar','Afar'), ('afr','Afrikaans'), ('amh','Amharic'), ('ara','Arabic'), ('asm','Assamese'), ('aym','Aymara'), ('aze','Azerbaijani'), ('bak','Bashkir'), ('ben','Bengali'), ('bih','Bihari'), ('bis','Bislama'), ('bre','Breton'), ('bul','Bulgarian'), ('bel','Byelorussian'), ('cat','Catalan'), ('cos','Corsican'), ('dan','Danish'), ('dzo','Dzongkha'), ('eng','English'), ('fra','French'), ('epo','Esperanto'), ('est','Estonian'), ('fao','Faroese'), ('fij','Fijian'), ('fin','Finnish'), ('fry','Frisian'), ('glg','Gallegan'), ('ger','German'), ('kal','Greenlandic'), ('grn','Guarani'), ('guj','Gujarati'), ('hau','Hausa'), ('heb','Hebrew'), ('hin','Hindi'), ('hun','Hungarian'), ('ind','Indonesian'), ('ina','Interlingua (International Auxiliary language Association)'), ('iku','Inuktitut'), ('ipk','Inupiak'), ('ita','Italian'), ('jpn','Japanese'), ('kan','Kannada'), ('kas','Kashmiri'), ('kaz','Kazakh'), ('khm','Khmer'), ('kin','Kinyarwanda'), ('kir','Kirghiz'), ('kor','Korean'), ('kur','Kurdish'), ('oci', b"Langue d 'Oc (post 1500)"), ('lao','Lao'), ('lat','Latin'), ('lav','Latvian'), ('lin','Lingala'), ('lit','Lithuanian'), ('mlg','Malagasy'), ('mlt','Maltese'), ('mar','Marathi'), ('mol','Moldavian'), ('mon','Mongolian'), ('nau','Nauru'), ('nep','Nepali'), ('nor','Norwegian'), ('ori','Oriya'), ('orm','Oromo'), ('pan','Panjabi'), ('pol','Polish'), ('por','Portuguese'), ('pus','Pushto'), ('que','Quechua'), ('roh','Rhaeto-Romance'), ('run','Rundi'), ('rus','Russian'), ('smo','Samoan'), ('sag','Sango'), ('san','Sanskrit'), ('scr','Serbo-Croatian'), ('sna','Shona'), ('snd','Sindhi'), ('sin','Singhalese'), ('ssw','Siswant'), ('slv','Slovenian'), ('som','Somali'), ('sot','Sotho'), ('spa','Spanish'), ('sun','Sudanese'), ('swa','Swahili'), ('tgl','Tagalog'), ('tgk','Tajik'), ('tam','Tamil'), ('tat','Tatar'), ('tel','Telugu'), ('tha','Thai'), ('tir','Tigrinya'), ('tog','Tonga (Nyasa)'), ('tso','Tsonga'), ('tsn','Tswana'), ('tur','Turkish'), ('tuk','Turkmen'), ('twi','Twi'), ('uig','Uighur'), ('ukr','Ukrainian'), ('urd','Urdu'), ('uzb','Uzbek'), ('vie','Vietnamese'), ('vol','Volap\xc3\xbck'), ('wol','Wolof'), ('xho','Xhosa'), ('yid','Yiddish'), ('yor','Yoruba'), ('zha','Zhuang'), ('zul','Zulu')])), + ('temporal_extent_start', models.DateTimeField(help_text='time period covered by the content of the dataset (start)', null=True, verbose_name='temporal extent start', blank=True)), + ('temporal_extent_end', models.DateTimeField(help_text='time period covered by the content of the dataset (end)', null=True, verbose_name='temporal extent end', blank=True)), + ('supplemental_information', models.TextField(default='No information provided', help_text='any other descriptive information about the dataset', verbose_name='supplemental information')), + ('data_quality_statement', models.TextField(help_text="general explanation of the data producer's knowledge about the lineage of a dataset", null=True, verbose_name='data quality statement', blank=True)), + ('bbox_x0', models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True)), + ('bbox_x1', models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True)), + ('bbox_y0', models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True)), + ('bbox_y1', models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True)), + ('srid', models.CharField(default='EPSG:4326', max_length=255)), + ('csw_typename', models.CharField(default='gmd:MD_Metadata', max_length=32, verbose_name='CSW typename')), + ('csw_schema', models.CharField(default='http://www.isotc211.org/2005/gmd', max_length=64, verbose_name='CSW schema')), + ('csw_mdsource', models.CharField(default='local', max_length=256, verbose_name='CSW source')), + ('csw_insert_date', models.DateTimeField(auto_now_add=True, verbose_name='CSW insert date', null=True)), + ('csw_type', models.CharField(default='dataset', max_length=32, verbose_name='CSW type', choices=[('series', 'series'), ('software', 'computer program or routine'), ('featureType', 'feature type'), ('model', 'copy or imitation of an existing or hypothetical object'), ('collectionHardware', 'collection hardware'), ('collectionSession', 'collection session'), ('nonGeographicDataset', 'non-geographic data'), ('propertyType', 'property type'), ('fieldSession', 'field session'), ('dataset', 'dataset'), ('service', 'service interfaces'), ('attribute', 'attribute class'), ('attributeType', 'characteristic of a feature'), ('tile', 'tile or spatial subset of geographic data'), ('feature', 'feature'), ('dimensionGroup', 'dimension group')])), + ('csw_anytext', models.TextField(null=True, verbose_name='CSW anytext', blank=True)), + ('csw_wkt_geometry', models.TextField(default='POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))', verbose_name='CSW WKT geometry')), + ('metadata_uploaded', models.BooleanField(default=False)), + ('metadata_xml', models.TextField(default='', null=True, blank=True)), + ('popular_count', models.IntegerField(default=0)), + ('share_count', models.IntegerField(default=0)), + ('featured', models.BooleanField(default=False, help_text='Should this resource be advertised in home page?', verbose_name='Featured')), + ('is_published', models.BooleanField(default=True, help_text='Should this resource be published and searchable?', verbose_name='Is Published')), + ('thumbnail_url', models.TextField(null=True, blank=True)), + ('detail_url', models.CharField(max_length=255, null=True, blank=True)), + ('rating', models.IntegerField(default=0, null=True, blank=True)), + ], + options={ + 'permissions': (('view_resourcebase', 'Can view resource'), ('change_resourcebase_permissions', 'Can change resource permissions'), ('download_resourcebase', 'Can download resource'), ('publish_resourcebase', 'Can publish resource'), ('change_resourcebase_metadata', 'Can change resource metadata')), + }, + bases=(geonode.security.models.PermissionLevelMixin, models.Model), + ), + migrations.CreateModel( + name='RestrictionCodeType', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('identifier', models.CharField(max_length=255, editable=False)), + ('description', models.TextField(max_length=255, editable=False)), + ('description_en', models.TextField(max_length=255, null=True, editable=False)), + ('gn_description', models.TextField(max_length=255, verbose_name='GeoNode description')), + ('gn_description_en', models.TextField(max_length=255, null=True, verbose_name='GeoNode description')), + ('is_choice', models.BooleanField(default=True)), + ], + options={ + 'ordering': ('identifier',), + 'verbose_name_plural': 'Metadata Restriction Code Types', + }, + ), + migrations.CreateModel( + name='SpatialRepresentationType', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('identifier', models.CharField(max_length=255, editable=False)), + ('description', models.CharField(max_length=255, editable=False)), + ('description_en', models.CharField(max_length=255, null=True, editable=False)), + ('gn_description', models.CharField(max_length=255, verbose_name='GeoNode description')), + ('gn_description_en', models.CharField(max_length=255, null=True, verbose_name='GeoNode description')), + ('is_choice', models.BooleanField(default=True)), + ], + options={ + 'ordering': ('identifier',), + 'verbose_name_plural': 'Metadata Spatial Representation Types', + }, + ), + migrations.CreateModel( + name='TopicCategory', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('identifier', models.CharField(default='location', max_length=255)), + ('description', models.TextField(default='')), + ('description_en', models.TextField(default='', null=True)), + ('gn_description', models.TextField(default='', null=True, verbose_name='GeoNode description')), + ('gn_description_en', models.TextField(default='', null=True, verbose_name='GeoNode description')), + ('is_choice', models.BooleanField(default=True)) + ], + options={ + 'ordering': ('identifier',), + 'verbose_name_plural': 'Metadata Topic Categories', + }, + ), + migrations.AddField( + model_name='resourcebase', + name='category', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='base.TopicCategory', + help_text='high-level geographic data thematic classification to assist in the grouping and search of available geographic data sets.', null=True), + ), + migrations.AddField( + model_name='resourcebase', + name='contacts', + field=models.ManyToManyField(to=settings.AUTH_USER_MODEL, through='base.ContactRole'), + ), + migrations.AddField( + model_name='resourcebase', + name='license', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='base.License', + help_text='license of the dataset', null=True, verbose_name='License'), + ), + migrations.AddField( + model_name='resourcebase', + name='owner', + field=models.ForeignKey(related_name='owned_resource', on_delete=models.CASCADE, + verbose_name='Owner', blank=True, to=settings.AUTH_USER_MODEL, null=True), + ), + migrations.AddField( + model_name='resourcebase', + name='polymorphic_ctype', + field=models.ForeignKey(related_name='polymorphic_base.resourcebase_set+', + on_delete=models.CASCADE, editable=False, to='contenttypes.ContentType', null=True), + ), + migrations.AddField( + model_name='resourcebase', + name='regions', + field=models.ManyToManyField(help_text='keyword identifies a location', to='base.Region', verbose_name='keywords region', blank=True), + ), + migrations.AddField( + model_name='resourcebase', + name='restriction_code_type', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='base.RestrictionCodeType', + help_text='limitation(s) placed upon the access or use of the data.', null=True, verbose_name='restrictions'), + ), + migrations.AddField( + model_name='resourcebase', + name='spatial_representation_type', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='base.SpatialRepresentationType', + help_text='method used to represent geographic information in the dataset.', null=True, verbose_name='spatial representation type'), + ), + migrations.AddField( + model_name='link', + name='resource', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='base.ResourceBase', null=True), + ), + migrations.AddField( + model_name='contactrole', + name='resource', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='base.ResourceBase', null=True), + ), + migrations.AlterUniqueTogether( + name='contactrole', + unique_together={('contact', 'resource', 'role')}, + ), + ] diff --git a/geonode/base/migrations/24_to_26.py b/geonode/base/migrations/24_to_26.py new file mode 100644 index 0000000..df7d89b --- /dev/null +++ b/geonode/base/migrations/24_to_26.py @@ -0,0 +1,124 @@ +from django.db import migrations, models +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '24_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Backup', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('identifier', models.CharField(max_length=255, editable=False)), + ('name', models.CharField(max_length=100)), + ('name_en', models.CharField(max_length=100, null=True)), + ('date', models.DateTimeField(auto_now_add=True)), + ('description', models.TextField(null=True, blank=True)), + ('description_en', models.TextField(null=True, blank=True)), + ('base_folder', models.CharField(max_length=100)), + ('location', models.TextField(null=True, blank=True)), + ], + options={ + 'ordering': ('date',), + 'verbose_name_plural': 'Backups', + }, + ), + migrations.CreateModel( + name='HierarchicalKeyword', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(unique=True, max_length=100, verbose_name='Name')), + ('slug', models.SlugField(unique=True, max_length=100, verbose_name='Slug')), + ('path', models.CharField(unique=True, max_length=255)), + ('depth', models.PositiveIntegerField()), + ('numchild', models.PositiveIntegerField(default=0)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='TaggedContentItem', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('content_object', models.ForeignKey(to='base.ResourceBase', on_delete=models.CASCADE)), + ('tag', models.ForeignKey(related_name='keywords', to='base.HierarchicalKeyword', on_delete=models.CASCADE)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Thesaurus', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('identifier', models.CharField(unique=True, max_length=255)), + ('title', models.CharField(max_length=255)), + ('date', models.CharField(default='', max_length=20)), + ('description', models.TextField(default='', max_length=255)), + ('slug', models.CharField(default='', max_length=64)), + ], + options={ + 'ordering': ('identifier',), + 'verbose_name_plural': 'Thesauri', + }, + ), + migrations.CreateModel( + name='ThesaurusKeyword', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('about', models.CharField(max_length=255, null=True, blank=True)), + ('alt_label', models.CharField(default='', max_length=255, null=True, blank=True)), + ('thesaurus', models.ForeignKey(related_name='thesaurus', to='base.Thesaurus', on_delete=models.CASCADE)), + ], + options={ + 'ordering': ('alt_label',), + 'verbose_name_plural': 'Thesaurus Keywords', + }, + ), + migrations.CreateModel( + name='ThesaurusKeywordLabel', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('lang', models.CharField(max_length=3)), + ('label', models.CharField(max_length=255)), + ('keyword', models.ForeignKey(related_name='keyword', to='base.ThesaurusKeyword', on_delete=models.CASCADE)), + ], + options={ + 'ordering': ('keyword', 'lang'), + 'verbose_name_plural': 'Labels', + }, + ), + migrations.AddField( + model_name='topiccategory', + name='fa_class', + field=models.CharField(default='fa-times', max_length=64), + ), + migrations.AddField( + model_name='resourcebase', + name='keywords', + field=taggit.managers.TaggableManager(to='base.HierarchicalKeyword', through='base.TaggedContentItem', blank=True, help_text='commonly used word(s) or formalised word(s) or phrase(s) used to describe the subject (space or comma-separated', verbose_name='keywords'), + ), + migrations.AddField( + model_name='resourcebase', + name='metadata_uploaded_preserve', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='resourcebase', + name='tkeywords', + field=models.ManyToManyField(help_text='formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject (space or comma-separated', to='base.ThesaurusKeyword', null=True, blank=True), + ), + migrations.AlterUniqueTogether( + name='thesauruskeywordlabel', + unique_together={('keyword', 'lang')}, + ), + migrations.AlterUniqueTogether( + name='thesauruskeyword', + unique_together={('thesaurus', 'alt_label')}, + ), + ] diff --git a/geonode/base/migrations/26_to_27.py b/geonode/base/migrations/26_to_27.py new file mode 100644 index 0000000..dd60a65 --- /dev/null +++ b/geonode/base/migrations/26_to_27.py @@ -0,0 +1,52 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '24_to_26'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='tkeywords', + field=models.ManyToManyField(help_text='formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject (space or comma-separated', to='base.ThesaurusKeyword', blank=True), + ), + migrations.AddField( + model_name='resourcebase', + name='group', + field=models.ForeignKey(blank=True, to='auth.Group', null=True, on_delete=models.CASCADE), + ), + + migrations.AddField( + model_name='resourcebase', + name='alternate', + field=models.CharField(max_length=128, null=True, blank=True), + ), + migrations.AddField( + model_name='region', + name='bbox_x0', + field=models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True), + ), + migrations.AddField( + model_name='region', + name='bbox_x1', + field=models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True), + ), + migrations.AddField( + model_name='region', + name='bbox_y0', + field=models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True), + ), + migrations.AddField( + model_name='region', + name='bbox_y1', + field=models.DecimalField(null=True, max_digits=19, decimal_places=10, blank=True), + ), + migrations.AddField( + model_name='region', + name='srid', + field=models.CharField(default='EPSG:4326', max_length=255), + ), + ] diff --git a/geonode/base/migrations/__init__.py b/geonode/base/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/base/models.py b/geonode/base/models.py new file mode 100644 index 0000000..01ca398 --- /dev/null +++ b/geonode/base/models.py @@ -0,0 +1,2129 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import re +import html +import math +import uuid +import logging +import traceback +from typing import List, Optional, Union, Tuple +from sequences.models import Sequence +from sequences import get_next_value +from PIL import Image + +from django.db import transaction +from django.db import models +from django.db.models import Max +from django.conf import settings +from django.utils.html import escape +from django.utils.timezone import now +from django.db.models import Q, signals +from django.db.utils import IntegrityError, OperationalError +from django.contrib.auth.models import Group +from django.core.files.base import ContentFile +from django.contrib.auth import get_user_model +from django.db.models.query import QuerySet +from django.db.models.fields.json import JSONField +from django.utils.functional import cached_property, classproperty +from django.contrib.gis.geos import GEOSGeometry, Polygon, Point +from django.contrib.gis.db.models import PolygonField +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ +from django.contrib.contenttypes.models import ContentType +from django.utils.html import strip_tags +from django.urls import reverse +from mptt.models import MPTTModel, TreeForeignKey + +from polymorphic.models import PolymorphicModel +from polymorphic.managers import PolymorphicManager + +from taggit.models import TagBase, ItemBase +from taggit.managers import TaggableManager, _TaggableManager + +from guardian.shortcuts import get_anonymous_user, get_objects_for_user +from treebeard.mp_tree import MP_Node, MP_NodeQuerySet, MP_NodeManager +from geonode import GeoNodeException + +from geonode.base import enumerations +from geonode.singleton import SingletonModel +from geonode.groups.conf import settings as groups_settings +from geonode.base.bbox_utils import BBOXHelper, polygon_from_bbox +from geonode.utils import ( + bbox_to_wkt, + find_by_attr, + bbox_to_projection, + bbox_swap, + get_allowed_extensions, + is_monochromatic_image, +) +from geonode.thumbs.utils import thumb_size, remove_thumbs, get_unique_upload_path, ThumbnailAlgorithms +from geonode.groups.models import GroupProfile +from geonode.security.utils import get_visible_resources, get_geoapp_subtypes +from geonode.security.models import PermissionLevelMixin +from geonode.security.permissions import VIEW_PERMISSIONS, OWNER_PERMISSIONS + +from geonode.notifications_helper import send_notification, get_notification_recipients +from geonode.people import Roles +from geonode.people.enumerations import ROLE_VALUES + +from urllib.parse import urlsplit, urljoin +from geonode.storage.manager import storage_manager + +logger = logging.getLogger(__name__) + + +class ContactRole(models.Model): + """ + ContactRole is an intermediate model to bind Profiles as Contacts to Resources and apply roles. + """ + + resource = models.ForeignKey("ResourceBase", blank=False, null=False, on_delete=models.CASCADE) + contact = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + role = models.CharField( + choices=ROLE_VALUES, max_length=255, help_text=_("function performed by the responsible " "party") + ) + + class Meta: + unique_together = (("contact", "resource", "role"),) + + +class TopicCategory(models.Model): + """ + Metadata about high-level geographic data thematic classification. + It should reflect a list of codes from TC211 + See: http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml + + """ + + identifier = models.CharField(max_length=255, default="location") + description = models.TextField(default="") + gn_description = models.TextField("GeoNode description", default="", null=True) + is_choice = models.BooleanField(default=True) + fa_class = models.CharField(max_length=64, default="fa-times") + + def __str__(self): + return self.gn_description + + class Meta: + ordering = ("identifier",) + verbose_name_plural = "Metadata Topic Categories" + + +class SpatialRepresentationType(models.Model): + """ + Metadata information about the spatial representation type. + It should reflect a list of codes from TC211 + See: http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml + + """ + + identifier = models.CharField(max_length=255, editable=False) + description = models.CharField(max_length=255, editable=False) + gn_description = models.CharField("GeoNode description", max_length=255) + is_choice = models.BooleanField(default=True) + + def __str__(self): + return str(self.gn_description) + + class Meta: + ordering = ("identifier",) + verbose_name_plural = "Metadata Spatial Representation Types" + + +class Region(MPTTModel): + code = models.CharField(max_length=50, unique=True) + name = models.CharField(max_length=255) + parent = TreeForeignKey("self", null=True, blank=True, on_delete=models.CASCADE, related_name="children") + + # Save bbox values in the database. + # This is useful for spatial searches and for generating thumbnail images + # and metadata records. + bbox_x0 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + bbox_x1 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + bbox_y0 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + bbox_y1 = models.DecimalField(max_digits=30, decimal_places=15, blank=True, null=True) + srid = models.CharField(max_length=30, blank=False, null=False, default="EPSG:4326") + + def __str__(self): + return str(self.name) + + @property + def bbox(self): + """BBOX is in the format: [x0,x1,y0,y1].""" + return [self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, self.srid] + + @property + def bbox_string(self): + """BBOX is in the format: [x0,y0,x1,y1].""" + return ",".join([str(self.bbox_x0), str(self.bbox_y0), str(self.bbox_x1), str(self.bbox_y1)]) + + @property + def geographic_bounding_box(self): + """BBOX is in the format: [x0,x1,y0,y1].""" + return bbox_to_wkt(self.bbox_x0, self.bbox_x1, self.bbox_y0, self.bbox_y1, srid=self.srid) + + @property + def geom(self): + srid, wkt = self.geographic_bounding_box.split(";") + srid = re.findall(r"\d+", srid) + geom = GEOSGeometry(wkt, srid=int(srid[0])) + geom.transform(4326) + return geom + + def is_assignable_to_geom(self, extent_geom: GEOSGeometry): + region_geom = self.geom + + if region_geom.contains(extent_geom) or region_geom.overlaps(extent_geom): + return True + return False + + class Meta: + ordering = ("name",) + verbose_name_plural = "Metadata Regions" + + class MPTTMeta: + order_insertion_by = ["name"] + + +class RestrictionCodeType(models.Model): + """ + Metadata information about the spatial representation type. + It should reflect a list of codes from TC211 + See: http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml + + """ + + identifier = models.CharField(max_length=255, editable=False) + description = models.TextField(max_length=255, editable=False) + gn_description = models.TextField("GeoNode description", max_length=255) + is_choice = models.BooleanField(default=True) + + def __str__(self): + return str(self.gn_description) + + class Meta: + ordering = ("identifier",) + verbose_name_plural = "Metadata Restriction Code Types" + + +class License(models.Model): + identifier = models.CharField(max_length=255, editable=False) + name = models.CharField(max_length=255) + abbreviation = models.CharField(max_length=20, null=True, blank=True) + description = models.TextField(null=True, blank=True) + url = models.URLField(max_length=2000, null=True, blank=True) + license_text = models.TextField(null=True, blank=True) + + def __str__(self): + return str(self.name) + + @property + def name_long(self): + if self.abbreviation is None or len(self.abbreviation) == 0: + return self.name + else: + return f"{self.name} ({self.abbreviation})" + + @property + def description_bullets(self): + if self.description is None or len(self.description) == 0: + return "" + else: + bullets = [] + lines = self.description.split("\n") + for line in lines: + bullets.append(f"+ {line}") + return bullets + + class Meta: + ordering = ("name",) + verbose_name_plural = "Licenses" + + +class HierarchicalKeywordQuerySet(MP_NodeQuerySet): + """QuerySet to automatically create a root node if `depth` not given.""" + + def create(self, **kwargs): + if "depth" not in kwargs: + return self.model.add_root(**kwargs) + return super().create(**kwargs) + + +class HierarchicalKeywordManager(MP_NodeManager): + def get_queryset(self): + return HierarchicalKeywordQuerySet(self.model).order_by("path") + + +class HierarchicalKeyword(TagBase, MP_Node): + node_order_by = ["name"] + objects = HierarchicalKeywordManager() + + @classmethod + def resource_keywords_tree(cls, user, parent=None, resource_type=None, resource_name=None): + """Returns resource keywords tree as a dict object.""" + user = user or get_anonymous_user() + resource_types = [resource_type] if resource_type else ["dataset", "map", "document"] + get_geoapp_subtypes() + qset = cls.get_tree(parent) + + if settings.SKIP_PERMS_FILTER: + resources = ResourceBase.objects.all() + else: + resources = get_objects_for_user(user, "base.view_resourcebase") + + resources = resources.filter( + polymorphic_ctype__model__in=resource_types, + ) + + if resource_name is not None: + resources = resources.filter(title=resource_name) + + resources = get_visible_resources( + resources, + user, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + tree = {} + + for hkw in qset.order_by("name"): + slug = hkw.slug + tags_count = 0 + + tags_count = TaggedContentItem.objects.filter(content_object__in=resources, tag=hkw).count() + + if tags_count > 0: + newobj = {"id": hkw.pk, "text": hkw.name, "href": slug, "tags": [tags_count]} + depth = hkw.depth or 1 + + # No use case, so purpose of 'parent' param is not clear. + # So following first 'if' statement is left unchanged + if (not parent and depth == 1) or (parent and depth == parent.depth): + if hkw.pk not in tree: + tree[hkw.pk] = newobj + tree[hkw.pk]["nodes"] = [] + else: + tree[hkw.pk]["tags"] = [tags_count] + else: + tree = cls._keywords_tree_of_a_child(hkw, tree, newobj) + + return list(tree.values()) + + @classmethod + def _keywords_tree_of_a_child(cls, child, tree, newobj): + qs = cls.get_tree(child.get_root()) + parent = qs[0] + + if parent.id not in tree: + tree[parent.id] = {"id": parent.id, "text": parent.name, "href": parent.slug, "tags": [], "nodes": []} + + node = tree[parent.id] + + for kw in qs: + if child.is_descendant_of(kw): + if kw.depth > 1: + item_found = None + if node["nodes"]: + item_found = find_by_attr(node["nodes"], kw.id) + + if item_found is None: + node["nodes"].append({"id": kw.id, "text": kw.name, "href": kw.slug, "nodes": []}) + node = node["nodes"][-1] + else: + node = item_found + node["nodes"] = getattr(node, "nodes", []) + + # All leaves appended but a child which is not a leaf may not be added + # again, as a leaf, but only its tag count be updated + item_found = find_by_attr(node["nodes"], newobj["id"]) + if item_found is not None: + item_found["tags"] = newobj["tags"] + else: + node["nodes"].append(newobj) + + return tree + + +class TaggedContentItem(ItemBase): + content_object = models.ForeignKey("ResourceBase", on_delete=models.CASCADE) + tag = models.ForeignKey("HierarchicalKeyword", related_name="keywords", on_delete=models.CASCADE) + + # see https://github.com/alex/django-taggit/issues/101 + @classmethod + def tags_for(cls, model, instance=None, **extra_filters): + kwargs = extra_filters or {} + if instance is not None: + return cls.tag_model().objects.filter(**{f"{cls.tag_relname()}__content_object": instance}, **kwargs) + return ( + cls.tag_model() + .objects.filter(**{f"{cls.tag_relname()}__content_object__isnull": False}, **kwargs) + .distinct() + ) + + +class _HierarchicalTagManager(_TaggableManager): + def add(self, *tags, through_defaults=None, tag_kwargs=None): + if tag_kwargs is None: + tag_kwargs = {} + + str_tags = set([t for t in tags if not isinstance(t, self.through.tag_model())]) + tag_objs = set(tags) - str_tags + # If str_tags has 0 elements Django actually optimizes that to not do a + # query. Malcolm is very smart. + try: + existing = self.through.tag_model().objects.filter(name__in=str_tags, **tag_kwargs).all() + tag_objs.update(existing) + new_ids = set() + _new_keywords = str_tags - set(t.name for t in existing) + for new_tag in list(_new_keywords): + new_tag = escape(new_tag) + new_tag_obj = None + with transaction.atomic(): + try: + new_tag_obj = HierarchicalKeyword.add_root(name=new_tag) + except Exception as e: + logger.exception(e) + # HierarchicalKeyword.add_root didn't return, probably the keyword already exists + if not new_tag_obj: + new_tag_obj = HierarchicalKeyword.objects.filter(name=new_tag) + if new_tag_obj.exists(): + new_tag_obj.first() + if new_tag_obj: + tag_objs.add(new_tag_obj) + new_ids.add(new_tag_obj.id) + else: + # Something has gone seriously wrong and we cannot assign the tag to the resource + logger.error(f"Error during the keyword creation for keyword: {new_tag}") + + signals.m2m_changed.send( + sender=self.through, + action="pre_add", + instance=self.instance, + reverse=False, + model=self.through.tag_model(), + pk_set=new_ids, + ) + + for tag in tag_objs: + try: + self.through.objects.get_or_create(tag=tag, **self._lookup_kwargs(), defaults=through_defaults) + except Exception as e: + logger.exception(e) + + signals.m2m_changed.send( + sender=self.through, + action="post_add", + instance=self.instance, + reverse=False, + model=self.through.tag_model(), + pk_set=new_ids, + ) + except IntegrityError as e: + logger.warning("The keyword provided already exists", exc_info=e) + except OperationalError as e: + logger.warning( + "An error has occured with the DB connection. Please try to re-add the keywords again", exc_info=e + ) + except Exception as e: + raise e + + +class Thesaurus(models.Model): + """ + Loadable thesaurus containing keywords in different languages + """ + + id = models.AutoField(null=False, blank=False, unique=True, primary_key=True) + + identifier = models.CharField(max_length=255, null=False, blank=False, unique=True) + + # read from the RDF file + title = models.CharField(max_length=255, null=False, blank=False) + # read from the RDF file + date = models.CharField(max_length=20, default="") + # read from the RDF file + description = models.TextField(max_length=255, default="") + + slug = models.CharField(max_length=64, default="", null=True, blank=True) + + about = models.CharField(max_length=255, null=True, blank=True) + + card_min = models.IntegerField(default=0) + card_max = models.IntegerField(default=-1) + facet = models.BooleanField(default=True) + order = models.IntegerField(null=False, default=0) + + def __str__(self): + return str(self.identifier) + + class Meta: + ordering = ("identifier",) + verbose_name_plural = "Thesauri" + + +class ThesaurusKeywordLabel(models.Model): + """ + Loadable thesaurus containing keywords in different languages + """ + + # read from the RDF file + lang = models.CharField(max_length=10) + # read from the RDF file + label = models.CharField(max_length=255) + # note = models.CharField(max_length=511) + + keyword = models.ForeignKey("ThesaurusKeyword", related_name="keyword", on_delete=models.CASCADE) + + def __str__(self): + return str(self.label) + + class Meta: + ordering = ("keyword", "lang") + verbose_name_plural = "Thesaurus Keyword Labels" + unique_together = (("keyword", "lang"),) + + +class ThesaurusKeyword(models.Model): + """ + Loadable thesaurus containing keywords in different languages + """ + + # read from the RDF file + about = models.CharField(max_length=255, null=True, blank=True) + # read from the RDF file + alt_label = models.CharField(max_length=255, default="", null=True, blank=True) + + thesaurus = models.ForeignKey("Thesaurus", related_name="thesaurus", on_delete=models.CASCADE) + + image = models.CharField(max_length=512, help_text="A URL to an image", null=True, blank=True) + + def __str__(self): + return str(self.alt_label) + + @property + def labels(self): + return ThesaurusKeywordLabel.objects.filter(keyword=self) + + class Meta: + ordering = ("alt_label",) + verbose_name_plural = "Thesaurus Keywords" + unique_together = (("thesaurus", "alt_label"),) + + +def generate_thesaurus_reference(instance, *args, **kwargs): + if instance.about: + return instance.about + + prefix = instance.thesaurus.about or f"{settings.SITEURL}/thesaurus/{instance.thesaurus.identifier}" + suffix = instance.alt_label or instance.id + instance.about = f"{prefix}#{suffix}" + + instance.save() + return instance.about + + +signals.post_save.connect(generate_thesaurus_reference, sender=ThesaurusKeyword) + + +class ThesaurusLabel(models.Model): + """ + Contains localized version of the thesaurus title + """ + + # read from the RDF file + lang = models.CharField(max_length=10) + # read from the RDF file + label = models.CharField(max_length=255) + + thesaurus = models.ForeignKey("Thesaurus", related_name="rel_thesaurus", on_delete=models.CASCADE) + + def __str__(self): + return str(self.label) + + class Meta: + ordering = ("lang",) + verbose_name_plural = "Thesaurus Labels" + unique_together = (("thesaurus", "lang"),) + + +class ResourceBaseManager(PolymorphicManager): + def admin_contact(self): + # this assumes there is at least one superuser + superusers = get_user_model().objects.filter(is_superuser=True).order_by("id") + if superusers.count() == 0: + raise RuntimeError("GeoNode needs at least one admin/superuser set") + + return superusers[0] + + def get_queryset(self): + return super().get_queryset().non_polymorphic() + + def polymorphic_queryset(self): + return super().get_queryset() + + @staticmethod + def upload_files(resource_id, files, force=False): + """Update the ResourceBase model""" + try: + out = [] + for f in files: + if force: + out.append(f) + elif os.path.isfile(f) and os.path.exists(f): + with open(f, "rb") as ff: + folder = os.path.basename(os.path.dirname(f)) + filename = os.path.basename(f) + file_uploaded_path = storage_manager.save(f"{folder}/{filename}", ff) + out.append(storage_manager.path(file_uploaded_path)) + + # making an update instead of save in order to avoid others + # signal like post_save and commiunication with geoserver + ResourceBase.objects.filter(id=resource_id).update(files=out) + return out + except Exception as e: + logger.exception(e) + + @staticmethod + def cleanup_uploaded_files(resource_id): + """Remove uploaded files, if any""" + from geonode.assets.utils import get_default_asset + + if ResourceBase.objects.filter(id=resource_id).exists(): + _resource = ResourceBase.objects.filter(id=resource_id).get() + asset = get_default_asset(_resource) # TODO: make sure to select the proper "uploaded" asset + if asset: + asset.delete() + + # Remove generated thumbnails, if any + filename = f"{_resource.get_real_instance().resource_type}-{_resource.get_real_instance().uuid}" + remove_thumbs(filename) + + # Remove the uploaded sessions, if any + if "geonode.upload" in settings.INSTALLED_APPS: + from geonode.upload.models import Upload + + # Need to call delete one by one in order to invoke the + # 'delete' overridden method + for upload in Upload.objects.filter(resource_id=_resource.get_real_instance().id): + upload.delete() + + +class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): + """ + Base Resource Object loosely based on ISO 19115:2003 + """ + + BASE_PERMISSIONS = { + "read": ["view_resourcebase"], + "write": ["change_resourcebase_metadata"], + "download": ["download_resourcebase"], + "owner": [ + "change_resourcebase", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", + ], + } + + PERMISSIONS = {} + + VALID_DATE_TYPES = [(x.lower(), _(x)) for x in ["Creation", "Publication", "Revision"]] + + abstract_help_text = _("brief narrative summary of the content of the resource(s)") + date_help_text = _("reference date for the cited resource") + date_type_help_text = _("identification of when a given event occurred") + edition_help_text = _("version of the cited resource") + attribution_help_text = _( + "authority or function assigned, as to a ruler, legislative assembly, delegate, or the like." + ) + doi_help_text = _("a DOI will be added by Admin before publication.") + purpose_help_text = _("summary of the intentions with which the resource(s) was developed") + maintenance_frequency_help_text = _( + "frequency with which modifications and deletions are made to the data after " "it is first produced" + ) + keywords_help_text = _( + "commonly used word(s) or formalised word(s) or phrase(s) used to describe the subject " + "(space or comma-separated)" + ) + tkeywords_help_text = _( + "formalised word(s) or phrase(s) from a fixed thesaurus used to describe the subject " + "(space or comma-separated)" + ) + regions_help_text = _("keyword identifies a location") + restriction_code_type_help_text = _("limitation(s) placed upon the access or use of the data.") + constraints_other_help_text = _( + "other restrictions and legal prerequisites for accessing and using the resource or" " metadata" + ) + license_help_text = _("license of the dataset") + language_help_text = _("language used within the dataset") + category_help_text = _( + "high-level geographic data thematic classification to assist in the grouping and search of " + "available geographic data sets." + ) + spatial_representation_type_help_text = _("method used to represent geographic information in the dataset.") + temporal_extent_start_help_text = _("time period covered by the content of the dataset (start)") + temporal_extent_end_help_text = _("time period covered by the content of the dataset (end)") + data_quality_statement_help_text = _( + "general explanation of the data producer's knowledge about the lineage of a" " dataset" + ) + extra_metadata_help_text = _( + 'Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]' + ) + # internal fields + uuid = models.CharField(max_length=36, unique=True, default=uuid.uuid4) + title = models.CharField(_("title"), max_length=255, help_text=_("name by which the cited resource is known")) + abstract = models.TextField(_("abstract"), max_length=2000, blank=True, help_text=abstract_help_text) + purpose = models.TextField(_("purpose"), max_length=500, null=True, blank=True, help_text=purpose_help_text) + owner = models.ForeignKey( + settings.AUTH_USER_MODEL, related_name="owned_resource", verbose_name=_("Owner"), on_delete=models.PROTECT + ) + contacts = models.ManyToManyField(settings.AUTH_USER_MODEL, through="ContactRole") + alternate = models.CharField(_("alternate"), max_length=255, null=True, blank=True) + date = models.DateTimeField(_("date"), default=now, help_text=date_help_text) + date_type = models.CharField( + _("date type"), max_length=255, choices=VALID_DATE_TYPES, default="publication", help_text=date_type_help_text + ) + edition = models.CharField(_("edition"), max_length=255, blank=True, null=True, help_text=edition_help_text) + attribution = models.CharField( + _("Attribution"), max_length=2048, blank=True, null=True, help_text=attribution_help_text + ) + doi = models.CharField(_("DOI"), max_length=255, blank=True, null=True, help_text=doi_help_text) + maintenance_frequency = models.CharField( + _("maintenance frequency"), + max_length=255, + choices=enumerations.UPDATE_FREQUENCIES, + blank=True, + null=True, + help_text=maintenance_frequency_help_text, + ) + keywords = TaggableManager( + _("keywords"), + through=TaggedContentItem, + blank=True, + help_text=keywords_help_text, + manager=_HierarchicalTagManager, + ) + tkeywords = models.ManyToManyField( + ThesaurusKeyword, verbose_name=_("keywords"), null=True, blank=True, help_text=tkeywords_help_text + ) + regions = models.ManyToManyField( + Region, verbose_name=_("keywords region"), null=True, blank=True, help_text=regions_help_text + ) + restriction_code_type = models.ForeignKey( + RestrictionCodeType, + verbose_name=_("restrictions"), + help_text=restriction_code_type_help_text, + null=True, + blank=True, + on_delete=models.SET_NULL, + limit_choices_to=Q(is_choice=True), + ) + constraints_other = models.TextField( + _("restrictions other"), blank=True, null=True, help_text=constraints_other_help_text + ) + license = models.ForeignKey( + License, + null=True, + blank=True, + verbose_name=_("License"), + help_text=license_help_text, + on_delete=models.SET_NULL, + ) + language = models.CharField( + _("language"), max_length=3, choices=enumerations.ALL_LANGUAGES, default="eng", help_text=language_help_text + ) + category = models.ForeignKey( + TopicCategory, + null=True, + blank=True, + on_delete=models.SET_NULL, + limit_choices_to=Q(is_choice=True), + help_text=category_help_text, + ) + spatial_representation_type = models.ForeignKey( + SpatialRepresentationType, + null=True, + blank=True, + on_delete=models.SET_NULL, + limit_choices_to=Q(is_choice=True), + verbose_name=_("spatial representation type"), + help_text=spatial_representation_type_help_text, + ) + + # Section 5 + temporal_extent_start = models.DateTimeField( + _("temporal extent start"), blank=True, null=True, help_text=temporal_extent_start_help_text + ) + temporal_extent_end = models.DateTimeField( + _("temporal extent end"), blank=True, null=True, help_text=temporal_extent_end_help_text + ) + supplemental_information = models.TextField( + _("supplemental information"), + max_length=2000, + default=enumerations.DEFAULT_SUPPLEMENTAL_INFORMATION, + help_text=_("any other descriptive information about the dataset"), + ) + + # Section 8 + data_quality_statement = models.TextField( + _("data quality statement"), max_length=2000, blank=True, null=True, help_text=data_quality_statement_help_text + ) + group = models.ForeignKey(Group, null=True, blank=True, on_delete=models.SET_NULL) + + # Section 9 + # see metadata_author property definition below + + # Save bbox values in the database. + # This is useful for spatial searches and for generating thumbnail images + # and metadata records. + bbox_polygon = PolygonField(null=True, blank=True) + ll_bbox_polygon = PolygonField(null=True, blank=True) + + srid = models.CharField(max_length=30, blank=False, null=False, default="EPSG:4326") + + # CSW specific fields + csw_typename = models.CharField(_("CSW typename"), max_length=32, default="gmd:MD_Metadata", null=False) + csw_schema = models.CharField( + _("CSW schema"), max_length=64, default="http://www.isotc211.org/2005/gmd", null=False + ) + csw_mdsource = models.CharField(_("CSW source"), max_length=256, default="local", null=False) + csw_insert_date = models.DateTimeField(_("CSW insert date"), auto_now_add=True, null=True) + csw_type = models.CharField( + _("CSW type"), max_length=32, default="dataset", null=False, choices=enumerations.HIERARCHY_LEVELS + ) + csw_anytext = models.TextField(_("CSW anytext"), null=True, blank=True) + csw_wkt_geometry = models.TextField( + _("CSW WKT geometry"), null=False, default="POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))" + ) + + # metadata XML specific fields + metadata_uploaded = models.BooleanField(default=False) + metadata_uploaded_preserve = models.BooleanField(_("Metadata uploaded preserve"), default=False) + metadata_xml = models.TextField( + null=True, default='', blank=True + ) + popular_count = models.IntegerField(default=0) + share_count = models.IntegerField(default=0) + featured = models.BooleanField( + _("Featured"), default=False, help_text=_("Should this resource be advertised in home page?") + ) + was_published = models.BooleanField(_("Was Published"), default=True, help_text=_("Previous Published state.")) + is_published = models.BooleanField( + _("Is Published"), default=True, help_text=_("Should this resource be published and searchable?") + ) + was_approved = models.BooleanField(_("Was Approved"), default=True, help_text=_("Previous Approved state.")) + is_approved = models.BooleanField( + _("Approved"), default=True, help_text=_("Is this resource validated from a publisher or editor?") + ) + advertised = models.BooleanField( + _("Advertised"), + default=True, + help_text=_("If False, will hide the resource from search results and catalog listings"), + ) + # fields necessary for the apis + thumbnail_url = models.TextField(_("Thumbnail url"), null=True, blank=True) + thumbnail_path = models.TextField(_("Thumbnail path"), null=True, blank=True) + rating = models.IntegerField(default=0, null=True, blank=True) + created = models.DateTimeField(auto_now_add=True, null=True, blank=True) + last_updated = models.DateTimeField(auto_now=True, null=True, blank=True) + + state = models.CharField( + _("State"), + max_length=16, + null=False, + blank=False, + default=enumerations.STATE_READY, + choices=enumerations.PROCESSING_STATES, + help_text=_("Hold the resource processing state."), + ) + + sourcetype = models.CharField( + _("Source Type"), + max_length=16, + null=False, + blank=False, + default=enumerations.SOURCE_TYPE_LOCAL, + choices=enumerations.SOURCE_TYPES, + help_text=_('The resource source type, which can be one of "LOCAL", "REMOTE" or "COPYREMOTE".'), + ) + + remote_typename = models.CharField( + _("Remote Service Typename"), + null=True, + blank=True, + max_length=512, + help_text=_("Name of the Remote Service if any."), + ) + + # fields controlling security state + dirty_state = models.BooleanField( + _("Dirty State"), default=False, help_text=_("Security Rules Are Not Synched with GeoServer!") + ) + + users_geolimits = models.ManyToManyField("UserGeoLimit", related_name="users_geolimits", null=True, blank=True) + + groups_geolimits = models.ManyToManyField("GroupGeoLimit", related_name="groups_geolimits", null=True, blank=True) + + resource_type = models.CharField(_("Resource Type"), max_length=1024, blank=True, null=True) + + metadata_only = models.BooleanField( + _("Metadata"), default=False, help_text=_("If true, will be excluded from search") + ) + + blob = JSONField(null=True, default=dict, blank=True) + + subtype = models.CharField(max_length=128, null=True, blank=True) + + metadata = models.ManyToManyField( + "ExtraMetadata", verbose_name=_("Extra Metadata"), null=True, blank=True, help_text=extra_metadata_help_text + ) + + objects = ResourceBaseManager() + + class Meta: + # custom permissions, + # add, change and delete are standard in django-guardian + permissions = ( + # ('view_resourcebase', 'Can view resource'), + ("change_resourcebase_permissions", "Can change resource permissions"), + ("download_resourcebase", "Can download resource"), + ("publish_resourcebase", "Can publish resource"), + ("change_resourcebase_metadata", "Can change resource metadata"), + ) + + def __init__(self, *args, **kwargs): + # Provide legacy support for bbox fields + try: + bbox = [kwargs.pop(key, None) for key in ("bbox_x0", "bbox_y0", "bbox_x1", "bbox_y1")] + if all(bbox): + kwargs["bbox_polygon"] = Polygon.from_bbox(bbox) + kwargs["ll_bbox_polygon"] = Polygon.from_bbox(bbox) + except Exception as e: + logger.exception(e) + super().__init__(*args, **kwargs) + + def __str__(self): + return str(self.title) + + def _remove_html_tags(self, attribute_str): + _attribute_str = attribute_str + try: + pattern = re.compile("<.*?>") + _attribute_str = html.unescape( + re.sub(pattern, "", attribute_str).replace("\n", " ").replace("\r", "").strip() + ) + except Exception: + if attribute_str: + _attribute_str = html.unescape(attribute_str.replace("\n", " ").replace("\r", "").strip()) + return strip_tags(_attribute_str) + + @classproperty + def allowed_permissions(cls): + return { + "anonymous": VIEW_PERMISSIONS, + "default": OWNER_PERMISSIONS, + groups_settings.REGISTERED_MEMBERS_GROUP_NAME: OWNER_PERMISSIONS, + } + + @classproperty + def compact_permission_labels(cls): + return { + "none": _("None"), + "view": _("View"), + "download": _("Download"), + "edit": _("Edit"), + "manage": _("Manage"), + "owner": _("Owner"), + } + + @property + def raw_abstract(self): + return self._remove_html_tags(self.abstract) + + @property + def raw_purpose(self): + return self._remove_html_tags(self.purpose) + + @property + def raw_constraints_other(self): + return self._remove_html_tags(self.constraints_other) + + @property + def raw_supplemental_information(self): + return self._remove_html_tags(self.supplemental_information) + + @property + def raw_data_quality_statement(self): + return self._remove_html_tags(self.data_quality_statement) + + @property + def detail_url(self): + return self.get_absolute_url() + + def clean(self): + if self.title: + self.title = self.title.replace(",", "_") + return super().clean() + + def save(self, notify=False, *args, **kwargs): + """ + Send a notification when a resource is created or updated + """ + if not self.resource_type and self.polymorphic_ctype and self.polymorphic_ctype.model: + self.resource_type = self.polymorphic_ctype.model.lower() + + # Resource Updated + _notification_sent = False + _group_status_changed = False + _approval_status_changed = False + + if hasattr(self, "class_name") and (self.pk is None or notify): + if self.pk is None and (self.title or getattr(self, "name", None)): + # Resource Created + if not self.title and getattr(self, "name", None): + self.title = getattr(self, "name", None) + notice_type_label = f"{self.class_name.lower()}_created" + recipients = get_notification_recipients(notice_type_label, resource=self) + send_notification(recipients, notice_type_label, {"resource": self}) + elif self.pk: + # Group has changed + _group_status_changed = self.group != ResourceBase.objects.get(pk=self.get_self_resource().pk).group + + # Approval Notifications Here + if self.was_approved != self.is_approved: + if not _notification_sent and not self.was_approved and self.is_approved: + # Send "approved" notification + notice_type_label = f"{self.class_name.lower()}_approved" + recipients = get_notification_recipients(notice_type_label, resource=self) + send_notification(recipients, notice_type_label, {"resource": self}) + _notification_sent = True + self.was_approved = self.is_approved + _approval_status_changed = True + + # Publishing Notifications Here + if self.was_published != self.is_published: + if not _notification_sent and not self.was_published and self.is_published: + # Send "published" notification + notice_type_label = f"{self.class_name.lower()}_published" + recipients = get_notification_recipients(notice_type_label, resource=self) + send_notification(recipients, notice_type_label, {"resource": self}) + _notification_sent = True + self.was_published = self.is_published + _approval_status_changed = True + + # Updated Notifications Here + if not _notification_sent: + notice_type_label = f"{self.class_name.lower()}_updated" + recipients = get_notification_recipients(notice_type_label, resource=self) + send_notification(recipients, notice_type_label, {"resource": self}) + + if self.pk is None: + _initial_value = ResourceBase.objects.aggregate(Max("pk"))["pk__max"] + if not _initial_value: + _initial_value = 1 + else: + _initial_value += 1 + _next_value = get_next_value("ResourceBase", initial_value=_initial_value) # type(self).__name__, + if _initial_value > _next_value: + Sequence.objects.filter(name="ResourceBase").update(last=_initial_value) + _next_value = _initial_value + + self.pk = self.id = _next_value + + if isinstance(self.uuid, uuid.UUID): + self.uuid = str(self.uuid) + elif not self.uuid or callable(self.uuid) or len(self.uuid) == 0: + self.uuid = str(uuid.uuid4()) + super().save(*args, **kwargs) + + # Update workflow permissions + if _approval_status_changed or _group_status_changed: + self.set_permissions( + approval_status_changed=_approval_status_changed, group_status_changed=_group_status_changed + ) + + def delete(self, notify=True, *args, **kwargs): + """ + Send a notification when a layer, map or document is deleted + """ + from geonode.resource.manager import resource_manager + + resource_manager.remove_permissions(self.uuid, instance=self.get_real_instance()) + + # delete assets. TODO: when standalone Assets will be allowed, only dependable Assets shall be removed + links_with_assets = Link.objects.filter(resource=self, asset__isnull=False).prefetch_related("asset") + for link in links_with_assets: + link.asset.delete() + + if hasattr(self, "class_name") and notify: + notice_type_label = f"{self.class_name.lower()}_deleted" + recipients = get_notification_recipients(notice_type_label, resource=self) + send_notification(recipients, notice_type_label, {"resource": self}) + + super().delete(*args, **kwargs) + + def get_upload_session(self): + raise NotImplementedError() + + @property + def site_url(self): + return settings.SITEURL + + @property + def creator(self): + return self.owner.get_full_name() or self.owner.username + + @property + def perms(self): + return [] + + @property + def organizationname(self): + return self.owner.organization + + @property + def restriction_code(self): + return self.restriction_code_type.gn_description if self.restriction_code_type else None + + @property + def topiccategory(self): + return self.category.identifier if self.category else None + + @property + def csw_crs(self): + return "EPSG:4326" + + @property + def group_name(self): + return str(self.group).encode("utf-8", "replace") if self.group else None + + @property + def bbox(self): + """BBOX is in the format: [x0, x1, y0, y1, srid].""" + if self.bbox_polygon: + match = re.match(r"^(EPSG:)?(?P\d{4,6})$", self.srid) + srid = int(match.group("srid")) if match else 4326 + bbox = BBOXHelper(self.bbox_polygon.extent) + return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, f"EPSG:{srid}"] + bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) + return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] + + @property + def ll_bbox(self): + """BBOX is in the format [x0, x1, y0, y1, "EPSG:srid"]. Provides backwards + compatibility after transition to polygons.""" + if self.ll_bbox_polygon: + _bbox = self.ll_bbox_polygon.extent + srid = self.ll_bbox_polygon.srid + return [_bbox[0], _bbox[2], _bbox[1], _bbox[3], f"EPSG:{srid}"] + bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) + return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] + + @property + def ll_bbox_string(self): + """WGS84 BBOX is in the format: [x0,y0,x1,y1].""" + if self.bbox_polygon: + bbox = BBOXHelper.from_xy(self.ll_bbox[:4]) + + return f"{bbox.xmin:.7f},{bbox.ymin:.7f},{bbox.xmax:.7f},{bbox.ymax:.7f}" + bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) + return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] + + @property + def bbox_string(self): + """BBOX is in the format: [x0, y0, x1, y1]. Provides backwards compatibility + after transition to polygons.""" + if self.bbox_polygon: + bbox = BBOXHelper.from_xy(self.bbox[:4]) + + return f"{bbox.xmin:.7f},{bbox.ymin:.7f},{bbox.xmax:.7f},{bbox.ymax:.7f}" + bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) + return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] + + @property + def bbox_helper(self): + if self.bbox_polygon: + return BBOXHelper(self.bbox_polygon.extent) + bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) + return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] + + @cached_property + def bbox_x0(self): + if self.bbox_polygon: + return self.bbox[0] + return None + + @cached_property + def bbox_x1(self): + if self.bbox_polygon: + return self.bbox[1] + return None + + @cached_property + def bbox_y0(self): + if self.bbox_polygon: + return self.bbox[2] + return None + + @cached_property + def bbox_y1(self): + if self.bbox_polygon: + return self.bbox[3] + return None + + @property + def geographic_bounding_box(self): + """ + Returns an EWKT representation of the bounding box in EPSG:4326 + """ + if self.ll_bbox_polygon: + bbox = polygon_from_bbox(self.ll_bbox_polygon.extent, 4326) + return str(bbox) + else: + bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) + return bbox_to_wkt(bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, srid="EPSG:4326") + + @property + def license_light(self): + a = [] + if not self.license: + return "" + if self.license.name is not None and (len(self.license.name) > 0): + a.append(self.license.name) + if self.license.url is not None and (len(self.license.url) > 0): + a.append(f"({self.license.url})") + return " ".join(a) + + @property + def license_verbose(self): + a = [] + if self.license.name_long is not None and (len(self.license.name_long) > 0): + a.append(f"{self.license.name_long}:") + if self.license.description is not None and (len(self.license.description) > 0): + a.append(self.license.description) + if self.license.url is not None and (len(self.license.url) > 0): + a.append(f"({self.license.url})") + return " ".join(a) + + @property + def metadata_completeness(self): + required_fields = [ + "abstract", + "category", + "data_quality_statement", + "date", + "date_type", + "language", + "license", + "regions", + "title", + ] + if self.restriction_code_type and self.restriction_code_type.identifier == "otherRestrictions": + required_fields.append("constraints_other") + filled_fields = [] + for required_field in required_fields: + field = getattr(self, required_field, None) + if field: + if required_field == "license": + if field.name == "Not Specified": + continue + if required_field == "regions": + if not field.all(): + continue + if required_field == "category": + if not field.identifier: + continue + filled_fields.append(field) + return f"{len(filled_fields) * 100 / len(required_fields)}%" + + @property + def instance_is_processed(self): + try: + if hasattr(self.get_real_instance(), "processed"): + return self.get_real_instance().processed + return False + except Exception: + return False + + @property + def is_copyable(self): + if self.resource_type == "dataset": + from geonode.assets.utils import get_default_asset + from geonode.geoserver.helpers import select_relevant_files + + asset = get_default_asset(self) # TODO: maybe we need to filter by original files + allowed_file = select_relevant_files(get_allowed_extensions(), asset.location) if asset else [] + return len(allowed_file) != 0 + return True + + def keyword_list(self): + return [kw.name for kw in self.keywords.all()] + + def keyword_slug_list(self): + return [kw.slug for kw in self.keywords.all()] + + def region_name_list(self): + return [region.name for region in self.regions.all()] + + def spatial_representation_type_string(self): + if hasattr(self.spatial_representation_type, "identifier"): + return self.spatial_representation_type.identifier + else: + if hasattr(self, "subtype"): + if self.subtype == "raster": + return "grid" + return "vector" + else: + return None + + def set_dirty_state(self): + if not self.dirty_state: + self.dirty_state = True + ResourceBase.objects.filter(id=self.id).update(dirty_state=True) + + def clear_dirty_state(self): + if self.dirty_state: + self.dirty_state = False + ResourceBase.objects.filter(id=self.id).update(dirty_state=False) + + def set_processing_state(self, state): + self.state = state + ResourceBase.objects.filter(id=self.id).update(state=state) + if state == enumerations.STATE_PROCESSED: + self.clear_dirty_state() + elif state == enumerations.STATE_INVALID: + self.set_dirty_state() + + @property + def processed(self): + return self.state == enumerations.STATE_PROCESSED and not self.dirty_state + + @property + def keyword_csv(self): + try: + keywords_qs = self.get_real_instance().keywords.all() + if keywords_qs: + return ",".join(kw.name for kw in keywords_qs) + else: + return "" + except Exception: + return "" + + def get_absolute_url(self): + from geonode.client.hooks import hookset + + try: + return ( + self.get_real_instance().get_absolute_url() + if self != self.get_real_instance() + else hookset.get_absolute_url(self) + ) + except Exception as e: + logger.exception(e) + return None + + def set_bbox_polygon(self, bbox, srid): + """ + Set `bbox_polygon` from bbox values. + + :param bbox: list or tuple formatted as + [xmin, ymin, xmax, ymax] + :param srid: srid as string (e.g. 'EPSG:4326' or '4326') + """ + try: + bbox_polygon = Polygon.from_bbox(bbox) + self.bbox_polygon = bbox_polygon.clone() + self.srid = srid + # This is a trick in order to avoid PostGIS reprojecting the bbox at save time + # by assuming the default geometries have 'EPSG:4326' as srid. + ResourceBase.objects.filter(id=self.id).update(bbox_polygon=self.bbox_polygon, srid=srid) + finally: + self.set_ll_bbox_polygon(bbox, srid=srid) + + def set_ll_bbox_polygon(self, bbox, srid="EPSG:4326"): + """ + Set `ll_bbox_polygon` from bbox values. + + :param bbox: list or tuple formatted as + [xmin, ymin, xmax, ymax] + :param srid: srid as string (e.g. 'EPSG:4326' or '4326') + """ + try: + bbox_polygon = Polygon.from_bbox(bbox) + if srid == 4326 or srid.upper() == "EPSG:4326": + self.ll_bbox_polygon = bbox_polygon + else: + match = re.match(r"^(EPSG:)?(?P\d{4,6})$", str(srid)) + bbox_polygon.srid = int(match.group("srid")) if match else 4326 + + # Adapt coords order from xmin,ymin,xmax,ymax to xmin,xmax,ymin,ymax + standard_extent = list(bbox_polygon.extent) + bbox_gn_order = bbox_swap(standard_extent) + [srid] + projected_bbox_gn_order = bbox_to_projection(bbox_gn_order) + projected_bbox = bbox_swap(projected_bbox_gn_order[:-1]) + + self.ll_bbox_polygon = Polygon.from_bbox(projected_bbox) + ResourceBase.objects.filter(id=self.id).update(ll_bbox_polygon=self.ll_bbox_polygon) + except Exception as e: + raise GeoNodeException(e) + + def set_bounds_from_bbox(self, bbox, srid): + """ + Calculate zoom level and center coordinates in mercator. + + :param bbox: BBOX is either a `geos.Pologyon` or in the + format: [x0, x1, y0, y1], which is: + [min lon, max lon, min lat, max lat] or + [xmin, xmax, ymin, ymax] + :type bbox: list + """ + if isinstance(bbox, Polygon): + self.set_bbox_polygon(bbox.extent, srid) + self.set_center_zoom() + return + elif isinstance(bbox, list): + self.set_bbox_polygon([bbox[0], bbox[2], bbox[1], bbox[3]], srid) + self.set_center_zoom() + return + + if not bbox or len(bbox) < 4: + raise ValidationError(f"Bounding Box cannot be empty {self.name} for a given resource") + if not srid: + raise ValidationError(f"Projection cannot be empty {self.name} for a given resource") + + self.srid = srid + self.set_bbox_polygon((bbox[0], bbox[2], bbox[1], bbox[3]), srid) + self.set_center_zoom() + + def set_center_zoom(self): + """ + Sets the center coordinates and zoom level in EPSG:4326 + """ + if self.ll_bbox_polygon and len(self.ll_bbox_polygon.centroid.coords) > 0: + bbox = self.ll_bbox_polygon.clone() + center_x, center_y = bbox.centroid.coords + center = Point(center_x, center_y, srid=4326) + self.center_x, self.center_y = center.coords + try: + ext = bbox.extent + width_zoom = math.log(360 / (ext[2] - ext[0]), 2) + height_zoom = math.log(360 / (ext[3] - ext[1]), 2) + self.zoom = math.ceil(min(width_zoom, height_zoom)) + except ZeroDivisionError: + pass + + def download_links(self): + """assemble download links for pycsw""" + links = [] + for link in self.link_set.all(): + if link.link_type == "metadata": # avoid recursion + continue + if link.link_type == "html": + links.append((self.title, "Web address (URL)", "WWW:LINK-1.0-http--link", link.url)) + elif link.link_type in ("OGC:WMS", "OGC:WFS", "OGC:WCS"): + links.append((self.title, link.name, link.link_type, link.url)) + else: + _link_type = "WWW:DOWNLOAD-1.0-http--download" + try: + _store_type = getattr(self.get_real_instance(), "subtype", None) + if _store_type and _store_type in ["tileStore", "remote"] and link.extension in ("html"): + _remote_service = getattr(self.get_real_instance(), "_remote_service", None) + if _remote_service: + _link_type = f"WWW:DOWNLOAD-{_remote_service.type}" + except Exception as e: + logger.exception(e) + description = f"{self.title} ({link.name} Format)" + links.append((self.title, description, _link_type, link.url)) + return links + + @property + def embed_url(self): + return ( + self.get_real_instance().embed_url + if self != self.get_real_instance() + else reverse("resourcebase_embed", kwargs={"resourcebaseid": self.pk}) + ) + + def get_tiles_url(self): + """Return URL for Z/Y/X mapping clients or None if it does not exist.""" + try: + tiles_link = self.link_set.get(name="Tiles") + except Link.DoesNotExist: + return None + else: + return tiles_link.url + + def get_legend(self): + """Return Link for legend or None if it does not exist.""" + try: + legends_link = self.link_set.filter(name="Legend") + except Link.DoesNotExist: + tb = traceback.format_exc() + logger.debug(tb) + return None + except Link.MultipleObjectsReturned: + tb = traceback.format_exc() + logger.debug(tb) + return None + else: + return legends_link + + def get_legend_url(self, style_name=None): + """Return URL for legend or None if it does not exist. + + The legend can be either an image (for Geoserver's WMS) + or a JSON object for ArcGIS. + """ + legend = self.get_legend() + + if legend is None: + return None + + if legend.exists(): + if not style_name: + return legend.first().url + else: + for _legend in legend: + if style_name in _legend.url: + return _legend.url + return None + + def get_ows_url(self): + """Return URL for OGC WMS server None if it does not exist.""" + try: + ows_link = self.link_set.get(name="OGC:WMS") + except Link.DoesNotExist: + return None + else: + return ows_link.url + + def get_thumbnail_url(self): + """Return a thumbnail url. + + It could be a local one if it exists, a remote one (WMS GetImage) for example + """ + _thumbnail_url = self.thumbnail_url + local_thumbnails = self.link_set.filter(name="Thumbnail") + remote_thumbnails = self.link_set.filter(name="Remote Thumbnail") + if local_thumbnails.exists(): + _thumbnail_url = local_thumbnails.first().url + elif remote_thumbnails.exists(): + _thumbnail_url = remote_thumbnails.first().url + return _thumbnail_url + + def has_thumbnail(self): + """Determine if the thumbnail object exists and an image exists""" + return self.link_set.filter(name="Thumbnail").exists() + + # Note - you should probably broadcast layer#post_save() events to ensure + # that indexing (or other listeners) are notified + def save_thumbnail(self, filename, image, thumbnail_algorithm=ThumbnailAlgorithms.fit, **kwargs): + upload_path = get_unique_upload_path(filename) + # force conversion to JPEG output file + upload_path = f"{os.path.splitext(upload_path)[0]}.jpg" + try: + # Check that the image is valid + if is_monochromatic_image(None, image): + if not self.thumbnail_url and not image: + raise Exception("Generated thumbnail image is blank") + else: + # Skip Image creation + image = None + + if image: + actual_name = storage_manager.save(upload_path, ContentFile(image)) + actual_file_name = os.path.basename(actual_name) + + if filename != actual_file_name: + upload_path = upload_path.replace(filename, actual_file_name) + url = storage_manager.url(upload_path) + try: + # Optimize the Thumbnail size and resolution + im = Image.open(storage_manager.open(actual_name)) + im = thumbnail_algorithm(im, **kwargs) + + # Saving the thumb into a temporary directory on file system + tmp_location = os.path.abspath(f"{settings.MEDIA_ROOT}/{upload_path}") + im.save(tmp_location, quality="high") + + with open(tmp_location, "rb+") as img: + # Saving the img via storage manager + storage_manager.save(upload_path, img) + + # If we use a remote storage, the local img is deleted + if tmp_location != storage_manager.path(upload_path): + os.remove(tmp_location) + except Exception as e: + logger.exception(e) + + # check whether it is an URI or not + parsed = urlsplit(url) + if not parsed.netloc: + # assuming is a relative path to current site + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + url = urljoin(site_url, url) + + if thumb_size(upload_path) == 0: + raise Exception("Generated thumbnail image is zero size") + + # should only have one 'Thumbnail' link + Link.objects.filter(resource=self, name="Thumbnail").delete() + obj, _created = Link.objects.get_or_create( + resource=self, + name="Thumbnail", + defaults=dict( + url=url, + extension="png", + mime="image/png", + link_type="image", + ), + ) + # Cleaning up the old stuff + if self.thumbnail_path and storage_manager.exists(self.thumbnail_path): + storage_manager.delete(self.thumbnail_path) + # Store the new url and path + self.thumbnail_url = url + self.thumbnail_path = upload_path + obj.url = url + obj.save() + ResourceBase.objects.filter(id=self.id).update(thumbnail_url=url, thumbnail_path=upload_path) + except Exception as e: + logger.error(f"Error when generating the thumbnail for resource {self.id}. ({e})") + try: + Link.objects.filter(resource=self, name="Thumbnail").delete() + except Exception as e: + logger.error(f"Error when generating the thumbnail for resource {self.id}. ({e})") + + def set_missing_info(self): + """Set default permissions and point of contacts. + + It is mandatory to call it from descendant classes + but hard to enforce technically via signals or save overriding. + """ + user = None + if self.owner: + user = self.owner + else: + try: + user = ResourceBase.objects.admin_contact().user + except Exception: + pass + + if user: + if len(self.poc) == 0: + self.poc = [user] + if len(self.metadata_author) == 0: + self.metadata_author = [user] + + from guardian.models import UserObjectPermission + + logger.debug("Checking for permissions.") + # True if every key in the get_all_level_info dict is empty. + no_custom_permissions = UserObjectPermission.objects.filter( + content_type=ContentType.objects.get_for_model(self.get_self_resource()), object_pk=str(self.pk) + ).exists() + + if not no_custom_permissions: + logger.debug("There are no permissions for this object, setting default perms.") + self.set_default_permissions(owner=user) + + def maintenance_frequency_title(self): + return [v for v in enumerations.UPDATE_FREQUENCIES if v[0] == self.maintenance_frequency][0][1].title() + + def language_title(self): + return [v for v in enumerations.ALL_LANGUAGES if v[0] == self.language][0][1].title() + + # Contact Roles + def add_missing_metadata_author_or_poc(self): + """ + Set metadata_author and/or point of contact (poc) to a resource when any of them is missing + """ + if len(self.metadata_author) == 0: + self.metadata_author = [self.owner] + if len(self.poc) == 0: + self.poc = [self.owner] + + @staticmethod + def get_multivalue_role_property_names() -> List[str]: + """returns list of property names for all contact roles able to + handle multiple profile_users + + Returns: + _type_: List(str) + _description: list of names + """ + return [role.name for role in Roles.get_multivalue_ones()] + + @staticmethod + def get_multivalue_required_role_property_names() -> List[str]: + """returns list of property names for all contact roles that are required + + Returns: + _type_: List(str) + _description: list of names + """ + return [role.name for role in (set(Roles.get_multivalue_ones()) & set(Roles.get_required_ones()))] + + @staticmethod + def get_ui_toggled_role_property_names() -> List[str]: + """returns list of property names for all contact roles that are toggled of in metadata_editor + + Returns: + _type_: List(str) + _description: list of names + """ + return [role.name for role in (set(Roles.get_toggled_ones()) & set(Roles.get_toggled_ones()))] + + # typing not possible due to: from geonode.base.forms import ResourceBaseForm; unable due to circular ... + def set_contact_roles_from_metadata_edit(self, resource_base_form) -> bool: + """gets a ResourceBaseForm and extracts the Contact Role elements from it + + Args: + resource_base_form (ResourceBaseForm): ResourceBaseForm with contact roles set + + Returns: + bool: true if all contact roles could be set, else false + """ + failed = False + for role in self.get_multivalue_role_property_names(): + try: + self.__setattr__(role, resource_base_form.cleaned_data[role]) + except AttributeError: + logger.warning(f"unable to set contact role {role} for {self} ...") + failed = True + return failed + + def __get_contact_role_elements__(self, role: str) -> Optional[List[settings.AUTH_USER_MODEL]]: + """general getter of for all contact roles except owner + + Args: + role (str): string coresponding to ROLE_VALUES in geonode/people/enumarations, defining which propery is requested + Returns: + Optional[List[settings.AUTH_USER_MODEL]]: returns the requested contact role from the database + """ + try: + contact_role = ContactRole.objects.filter(role=role, resource=self) + contacts = [cr.contact for cr in contact_role] + except ContactRole.DoesNotExist: + contacts = None + return contacts + + # types allowed as input for Contact role properties + CONTACT_ROLE_USER_PROFILES_ALLOWED_TYPES = Union[settings.AUTH_USER_MODEL, QuerySet, List[settings.AUTH_USER_MODEL]] + + def __set_contact_role_element__(self, user_profile: CONTACT_ROLE_USER_PROFILES_ALLOWED_TYPES, role: str): + """general setter for all contact roles except owner in resource base + + Args: + user_profile (CONTACT_ROLE_USER_PROFILES_ALLOWED_TYPES): _description_ + role (str): tring coresponding to ROLE_VALUES in geonode/people/enumarations, defining which propery is to set + """ + + def __create_role__( + resource, role: str, user_profile: settings.AUTH_USER_MODEL + ) -> List[settings.AUTH_USER_MODEL]: + return ContactRole.objects.create(role=role, resource=resource, contact=user_profile) + + if isinstance(user_profile, QuerySet): + ContactRole.objects.filter(role=role, resource=self).delete() + return [__create_role__(self, role, user) for user in user_profile] + + elif isinstance(user_profile, get_user_model()): + ContactRole.objects.filter(role=role, resource=self).delete() + return __create_role__(self, role, user_profile) + + elif isinstance(user_profile, list) and all( + get_user_model().objects.filter(username=x).exists() for x in user_profile + ): + ContactRole.objects.filter(role=role, resource=self).delete() + return [ + __create_role__(self, role, user) for user in get_user_model().objects.filter(username__in=user_profile) + ] + + elif user_profile is None: + ContactRole.objects.filter(role=role, resource=self).delete() + else: + logger.error(f"Bad profile format for role: {role} ...") + + def get_defined_multivalue_contact_roles(self) -> List[Tuple[List[settings.AUTH_USER_MODEL], str]]: + """Returns all set contact roles of the ressource with additional ROLE_VALUES from geonode.people.enumarations.ROLE_VALUES. Mainly used to generate output xml more easy. + + Returns: + _type_: List[Tuple[List[people object], roles_label_name]] + _description: list tuples including two elements: 1. list of people have a certain role. 2. role label + """ + return { + role.label: self.__getattribute__(role.name) + for role in Roles.get_multivalue_ones() + if self.__getattribute__(role.name) + } + + def get_first_contact_of_role(self, role: str) -> Optional[ContactRole]: + """ + Get the first contact from the specified role. + + Parameters: + role (str): The role of the contact. + + Returns: + ContactRole or None: The first contact with the specified role, or None if not found. + """ + if contact := self.__get_contact_role_elements__(role): + return contact[0] + else: + return None + + # Contact Role: POC (pointOfContact) + def __get_poc__(self) -> List[settings.AUTH_USER_MODEL]: + return self.__get_contact_role_elements__(role="pointOfContact") + + def __set_poc__(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role="pointOfContact") + + poc = property(__get_poc__, __set_poc__) + + @property + def poc_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.poc) + + # Contact Role: metadata_author + def _get_metadata_author(self): + return self.__get_contact_role_elements__(role="author") + + def _set_metadata_author(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role="author") + + metadata_author = property(_get_metadata_author, _set_metadata_author) + + @property + def metadata_author_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.metadata_author) + + # Contact Role: PROCESSOR + def _get_processor(self): + return self.__get_contact_role_elements__(role=Roles.PROCESSOR.name) + + def _set_processor(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.PROCESSOR.name) + + processor = property(_get_processor, _set_processor) + + @property + def processor_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.processor) + + # Contact Role: PUBLISHER + def _get_publisher(self): + return self.__get_contact_role_elements__(role=Roles.PUBLISHER.name) + + def _set_publisher(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.PUBLISHER.name) + + publisher = property(_get_publisher, _set_publisher) + + @property + def publisher_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.publisher) + + # Contact Role: CUSTODIAN + def _get_custodian(self): + return self.__get_contact_role_elements__(role=Roles.CUSTODIAN.name) + + def _set_custodian(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.CUSTODIAN.name) + + custodian = property(_get_custodian, _set_custodian) + + @property + def custodian_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.custodian) + + # Contact Role: DISTRIBUTOR + def _get_distributor(self): + return self.__get_contact_role_elements__(role=Roles.DISTRIBUTOR.name) + + def _set_distributor(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.DISTRIBUTOR.name) + + distributor = property(_get_distributor, _set_distributor) + + @property + def distributor_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.distributor) + + # Contact Role: RESOURCE_USER + def _get_resource_user(self): + return self.__get_contact_role_elements__(role=Roles.RESOURCE_USER.name) + + def _set_resource_user(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.RESOURCE_USER.name) + + resource_user = property(_get_resource_user, _set_resource_user) + + @property + def resource_user_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.resource_user) + + # Contact Role: RESOURCE_PROVIDER + def _get_resource_provider(self): + return self.__get_contact_role_elements__(role=Roles.RESOURCE_PROVIDER.name) + + def _set_resource_provider(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.RESOURCE_PROVIDER.name) + + resource_provider = property(_get_resource_provider, _set_resource_provider) + + @property + def resource_provider_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.resource_provider) + + # Contact Role: ORIGINATOR + def _get_originator(self): + return self.__get_contact_role_elements__(role=Roles.ORIGINATOR.name) + + def _set_originator(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.ORIGINATOR.name) + + originator = property(_get_originator, _set_originator) + + @property + def originator_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.originator) + + # Contact Role: PRINCIPAL_INVESTIGATOR + def _get_principal_investigator(self): + return self.__get_contact_role_elements__(role=Roles.PRINCIPAL_INVESTIGATOR.name) + + def _set_principal_investigator(self, user_profile): + return self.__set_contact_role_element__(user_profile=user_profile, role=Roles.PRINCIPAL_INVESTIGATOR.name) + + principal_investigator = property(_get_principal_investigator, _set_principal_investigator) + + @property + def principal_investigator_csv(self): + return ",".join(p.get_full_name() or p.username for p in self.principal_investigator) + + def get_linked_resources(self, as_target: bool = False): + """ + Get all the linked resources to this ResourceBase instance. + This is implemented as a method so that derived classes can override it (for instance, Maps may add + related datasets) + """ + return ( + LinkedResource.get_linked_resources(target=self) + if as_target + else LinkedResource.get_linked_resources(source=self) + ) + + +class LinkManager(models.Manager): + """Helper class to access links grouped by type""" + + def data(self): + return self.get_queryset().filter(link_type="data") + + def image(self): + return self.get_queryset().filter(link_type="image") + + def download(self): + return self.get_queryset().filter(link_type__in=["image", "data", "original"]) + + def metadata(self): + return self.get_queryset().filter(link_type="metadata") + + def original(self): + return self.get_queryset().filter(link_type="original") + + def ows(self): + return self.get_queryset().filter(link_type__in=["OGC:WMS", "OGC:WFS", "OGC:WCS"]) + + +class LinkedResource(models.Model): + source = models.ForeignKey( + ResourceBase, related_name="linked_to", blank=False, null=False, on_delete=models.CASCADE + ) + target = models.ForeignKey(ResourceBase, related_name="linked_by", blank=True, null=False, on_delete=models.CASCADE) + internal = models.BooleanField(null=False, default=False) + + @classmethod + def get_linked_resources(cls, source: ResourceBase = None, target: ResourceBase = None, is_internal: bool = None): + if source is None and target is None: + raise ValueError("Both source and target filters missing") + + qs = LinkedResource.objects + if source: + qs = qs.filter(source=source).select_related("target") + if target: + qs = qs.filter(target=target).select_related("source") + if is_internal is not None: + qs = qs.filter(internal=is_internal) + return qs + + @classmethod + def get_target_ids(cls, source: ResourceBase, is_internal: bool = None): + sub = LinkedResource.objects.filter(source=source).values_list("target_id", flat=True) + if is_internal is not None: + sub = sub.filter(internal=is_internal) + return sub + + @classmethod + def get_targets(cls, source: ResourceBase, is_internal: bool = None): + return ResourceBase.objects.filter(id__in=cls.get_target_ids(source, is_internal)) + + @classmethod + def resolve_targets(cls, linked_resources): + return (lr.target for lr in linked_resources) + + @classmethod + def resolve_sources(cls, linked_resources): + return (lr.source for lr in linked_resources) + + +class Link(models.Model): + """Auxiliary model for storing links for resources. + + This helps avoiding the need for runtime lookups + to the OWS server or the CSW Catalogue. + + There are four types of links: + * original: For uploaded files (Shapefiles or GeoTIFFs) + * data: For WFS and WCS links that allow access to raw data + * image: For WMS and TMS links + * metadata: For CSW links + * OGC:WMS: for WMS service links + * OGC:WFS: for WFS service links + * OGC:WCS: for WCS service links + """ + + resource = models.ForeignKey(ResourceBase, blank=True, null=True, on_delete=models.CASCADE) + extension = models.CharField(max_length=255, help_text=_('For example "kml"')) + link_type = models.CharField(max_length=255, choices=[(x, x) for x in enumerations.LINK_TYPES]) + name = models.CharField(max_length=255, help_text=_('For example "View in Google Earth"')) + mime = models.CharField(max_length=255, help_text=_('For example "text/xml"')) + url = models.TextField(max_length=1000) + asset = models.ForeignKey("assets.Asset", null=True, on_delete=models.CASCADE) + + objects = LinkManager() + + def __str__(self): + return f"{self.link_type} link" + + +class MenuPlaceholder(models.Model): + name = models.CharField(max_length=255, null=False, blank=False, unique=True) + + def __str__(self): + return str(self.name) + + +class Menu(models.Model): + title = models.CharField(max_length=255, null=False, blank=False) + placeholder = models.ForeignKey(to="MenuPlaceholder", on_delete=models.CASCADE, null=False) + order = models.IntegerField( + null=False, + ) + + def __str__(self): + return str(self.title) + + class Meta: + unique_together = ( + ("placeholder", "order"), + ("placeholder", "title"), + ) + ordering = ["order"] + + +class MenuItem(models.Model): + title = models.CharField(max_length=255, null=False, blank=False) + menu = models.ForeignKey(to="Menu", null=False, on_delete=models.CASCADE) + order = models.IntegerField(null=False) + blank_target = models.BooleanField() + url = models.CharField(max_length=2000, null=False, blank=False) + + def __eq__(self, other): + return self.order == other.order + + def __ne__(self, other): + return self.order != other.order + + def __lt__(self, other): + return self.order < other.order + + def __le__(self, other): + return self.order <= other.order + + def __gt__(self, other): + return self.order > other.order + + def __ge__(self, other): + return self.order >= other.order + + def __hash__(self): + return hash(self.url) + + def __str__(self): + return str(self.title) + + class Meta: + unique_together = ( + ("menu", "order"), + ("menu", "title"), + ) + ordering = ["order"] + + +class Configuration(SingletonModel): + """ + A model used for managing the Geonode instance's global configuration, + without a need for reloading the instance. + + Usage: + from geonode.base.models import Configuration + config = Configuration.load() + """ + + read_only = models.BooleanField(default=False) + maintenance = models.BooleanField(default=False) + + class Meta: + verbose_name_plural = "Configuration" + + def __str__(self): + return "Configuration" + + +class UserGeoLimit(models.Model): + user = models.ForeignKey(settings.AUTH_USER_MODEL, null=False, blank=False, on_delete=models.CASCADE) + resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) + wkt = models.TextField(db_column="wkt", blank=True) + + +class GroupGeoLimit(models.Model): + group = models.ForeignKey(GroupProfile, null=False, blank=False, on_delete=models.CASCADE) + resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) + wkt = models.TextField(db_column="wkt", blank=True) + + +class ExtraMetadata(models.Model): + resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE) + metadata = JSONField(null=True, default=dict, blank=True) diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py new file mode 100644 index 0000000..d8cc78a --- /dev/null +++ b/geonode/base/populate_test_data.py @@ -0,0 +1,527 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +import os.path + +from io import BytesIO +from uuid import uuid4 +from itertools import cycle +from taggit.models import Tag +from taggit.models import TaggedItem +from datetime import datetime, timedelta + +from django.db import transaction +from django.utils import timezone +from django.db.utils import IntegrityError +from django.contrib.gis.geos import Polygon +from django.core.serializers import serialize +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission, Group +from django.core.files.uploadedfile import SimpleUploadedFile + +from geonode.assets.utils import create_asset_and_link +from geonode.maps.models import Map +from geonode.base import enumerations +from geonode.layers.models import Dataset +from geonode.compat import ensure_string +from geonode.documents.models import Document +from geonode.base.models import ResourceBase, TopicCategory +from geonode.geoapps.models import GeoApp + +# This is used to populate the database with the search fixture data. This is +# primarily used as a first step to generate the json data for the fixture using +# django's dumpdata + +logger = logging.getLogger(__name__) + +imgfile = BytesIO( + b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" +) +f = SimpleUploadedFile("test_img_file.gif", imgfile.read(), "image/gif") +dfile = [f"{os.path.dirname(__file__)}/tests/data/img.gif"] + + +def all_public(): + """ensure all layers, maps and documents are publicly available""" + for lyr in Dataset.objects.all(): + lyr.set_default_permissions() + lyr.clear_dirty_state() + lyr.set_processing_state(enumerations.STATE_PROCESSED) + for mp in Map.objects.all(): + mp.set_default_permissions() + mp.clear_dirty_state() + mp.set_processing_state(enumerations.STATE_PROCESSED) + for doc in Document.objects.all(): + doc.set_default_permissions() + doc.clear_dirty_state() + doc.set_processing_state(enumerations.STATE_PROCESSED) + ResourceBase.objects.all().update(dirty_state=False) + + +def create_fixtures(): + biota = TopicCategory.objects.get(identifier="biota") + location = TopicCategory.objects.get(identifier="location") + elevation = TopicCategory.objects.get(identifier="elevation") + farming = TopicCategory.objects.get(identifier="farming") + world_extent = [-180, 180, -90, 90] + + map_data = [ + ("GeoNode Default Map", "GeoNode default map abstract", ("populartag",), world_extent, biota), + ("ipsum lorem", "common ipsum lorem", ("populartag", "maptagunique"), world_extent, biota), + ("lorem1 ipsum1", "common abstract1", ("populartag",), world_extent, biota), + ("ipsum foo", "common bar lorem", ("populartag",), world_extent, location), + ("map one", "common this is a unique thing", ("populartag",), [0, 1, 0, 1], location), + ("quux", "common double thing", ("populartag",), [0, 5, 0, 5], location), + ("morx", "common thing double", ("populartag",), [0, 10, 0, 10], elevation), + ("titledupe something else ", "whatever common", ("populartag",), [0, 10, 0, 10], elevation), + ("something titledupe else ", "bar common", ("populartag",), [0, 50, 0, 50], elevation), + ("map metadata true", "map metadata true", ("populartag",), [0, 22, 0, 22], farming), + ] + + user_data = [ + ("bobby", "bob", "bobby", ""), + ("norman", "norman", "norman", ""), + ("user1", "pass", "uniquefirst", "foo"), + ("user2", "pass", "foo", "uniquelast"), + ("unique_username", "pass", "foo", "uniquelast"), + ("jblaze", "pass", "johnny", "blaze"), + ("foo", "pass", "bar", "baz"), + ] + + people_data = [ + ("this contains all my interesting profile information",), + ("some other information goes here",), + ] + now = datetime.now(timezone.get_current_timezone()) + step = timedelta(days=60) + + def get_test_date(): + def it(): + current = now - step + while True: + yield current + current = current - step + + itinst = it() + + def callable(): + return next(itinst) + + return callable + + next_date = get_test_date() + + dataset_data = [ + ("CA", "abstract1", "CA", "geonode:CA", world_extent, next_date(), ("populartag", "here"), elevation), + ("layer2", "abstract2", "layer2", "geonode:layer2", world_extent, next_date(), ("populartag",), elevation), + ( + "uniquetitle", + "something here", + "mylayer", + "geonode:mylayer", + world_extent, + next_date(), + ("populartag",), + elevation, + ), + ( + "common blar", + "lorem ipsum", + "foo", + "geonode:foo", + world_extent, + next_date(), + ("populartag", "layertagunique"), + location, + ), + ( + "common double it", + "whatever", + "whatever", + "geonode:whatever", + [0, 1, 0, 1], + next_date(), + ("populartag",), + location, + ), + ("common double time", "else", "fooey", "geonode:fooey", [0, 5, 0, 5], next_date(), ("populartag",), location), + ("common bar", "uniqueabstract", "quux", "geonode:quux", [0, 10, 0, 10], next_date(), ("populartag",), biota), + ("common morx", "lorem ipsum", "fleem", "geonode:fleem", [0, 50, 0, 50], next_date(), ("populartag",), biota), + ( + "dataset metadata true", + "lorem ipsum", + "fleem", + "geonode:metadatatrue", + [0, 22, 0, 22], + next_date(), + ("populartag",), + farming, + ), + ] + + document_data = [ + ("lorem ipsum", "common lorem ipsum", ("populartag",), world_extent, biota), + ("ipsum lorem", "common ipsum lorem", ("populartag", "doctagunique"), world_extent, biota), + ("lorem1 ipsum1", "common abstract1", ("populartag",), world_extent, biota), + ("ipsum foo", "common bar lorem", ("populartag",), world_extent, location), + ("doc one", "common this is a unique thing", ("populartag",), [0, 1, 0, 1], location), + ("quux", "common double thing", ("populartag",), [0, 5, 0, 5], location), + ("morx", "common thing double", ("populartag",), [0, 10, 0, 10], elevation), + ("titledupe something else ", "whatever common", ("populartag",), [0, 10, 0, 10], elevation), + ("something titledupe else ", "bar common", ("populartag",), [0, 50, 0, 50], elevation), + ("doc metadata true", "doc metadata true", ("populartag",), [0, 22, 0, 22], farming), + ] + + return map_data, user_data, people_data, dataset_data, document_data + + +def create_models(type=None, integration=False): + users = [] + obj_ids = [] + with transaction.atomic(): + map_data, user_data, people_data, dataset_data, document_data = create_fixtures() + registeredmembers_group, _ = Group.objects.get_or_create(name="registered-members") + anonymous_group, _ = Group.objects.get_or_create(name="anonymous") + cont_group, _ = Group.objects.get_or_create(name="contributors") + perm = Permission.objects.get(codename="add_resourcebase") + cont_group.permissions.add(perm) + logger.debug("[SetUp] Get or create user admin") + u, _ = get_user_model().objects.get_or_create(username="admin") + u.set_password("admin") + u.is_superuser = True + u.first_name = "admin" + u.save() + u.groups.add(anonymous_group) + users.append(u) + + for ud, pd in zip(user_data, cycle(people_data)): + user_name, password, first_name, last_name = ud + logger.debug(f"[SetUp] Get or create user {user_name}") + u, _ = get_user_model().objects.get_or_create(username=user_name) + u.set_password(password) + u.first_name = first_name + u.last_name = last_name + u.save() + u.groups.add(anonymous_group) + + if not (u.is_superuser or u.is_staff or u.is_anonymous): + u.groups.add(cont_group) + users.append(u) + + logger.debug(f"[SetUp] Add group {anonymous_group}") + get_user_model().objects.get(username="AnonymousUser").groups.add(anonymous_group) + + from geonode.utils import DisableDjangoSignals + + with DisableDjangoSignals(skip=integration): + if not type or ensure_string(type) == "map": + for md, user in zip(map_data, cycle(users)): + title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), category = md + logger.debug(f"[SetUp] Add map {title}") + m, _ = Map.objects.get_or_create( + title=title, + defaults=dict( + uuid=str(uuid4()), + abstract=abstract, + owner=user, + bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + srid="EPSG:4326", + category=category, + metadata_only=title == "map metadata true", + ), + ) + m.set_default_permissions(owner=user) + m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) + obj_ids.append(m.id) + for kw in kws: + m.keywords.add(kw) + m.save() + + if not type or ensure_string(type) == "document": + for dd, user in zip(document_data, cycle(users)): + title, abstract, kws, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), category = dd + logger.debug(f"[SetUp] Add document {title}") + m, _ = Document.objects.get_or_create( + title=title, + defaults=dict( + uuid=str(uuid4()), + abstract=abstract, + owner=user, + bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + srid="EPSG:4326", + # files=dfile, + extension="gif", + metadata_only=title == "doc metadata true", + ), + ) + _, _ = create_asset_and_link(m, m.owner, dfile) + m.set_default_permissions(owner=user) + m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) + obj_ids.append(m.id) + for kw in kws: + m.keywords.add(kw) + m.save() + + if not type or ensure_string(type) == "dataset": + for ld, user, subtype in zip(dataset_data, cycle(users), cycle(("raster", "vector"))): + title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws, category = ld + end = start + timedelta(days=365) + logger.debug(f"[SetUp] Add dataset {title}") + try: + dataset, _ = Dataset.objects.get_or_create( + alternate=alternate, + defaults=dict( + title=title, + abstract=abstract, + name=name, + bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + srid="EPSG:4326", + uuid=str(uuid4()), + owner=user, + temporal_extent_start=start, + temporal_extent_end=end, + date=start, + subtype=subtype, + category=category, + metadata_only=title == "dataset metadata true", + ), + ) + dataset.set_default_permissions(owner=user) + dataset.clear_dirty_state() + dataset.set_processing_state(enumerations.STATE_PROCESSED) + obj_ids.append(dataset.id) + for kw in kws: + dataset.keywords.add(kw) + dataset.save() + except IntegrityError: + pass + return obj_ids + + +def remove_models(obj_ids, type=None, integration=False): + from geonode.utils import DisableDjangoSignals + + with DisableDjangoSignals(skip=integration): + if not type: + remove_models(None, type=b"map") + remove_models(None, type=b"dataset") + remove_models(None, type=b"document") + if type == "map": + try: + m_ids = obj_ids or [mp.id for mp in Map.objects.all()] + for id in m_ids: + m = Map.objects.get(pk=id) + m.delete() + except Exception: + pass + elif type == "dataset": + try: + l_ids = obj_ids or [lyr.id for lyr in Dataset.objects.all()] + for id in l_ids: + dataset = Dataset.objects.get(pk=id) + dataset.delete() + except Exception: + pass + elif type == "document": + try: + d_ids = obj_ids or [doc.id for doc in Document.objects.all()] + for id in d_ids: + d = Document.objects.get(pk=id) + d.delete() + except Exception: + pass + + +def dump_models(path=None): + result = serialize( + "json", + sum( + [ + list(x) + for x in [ + get_user_model().objects.all(), + Dataset.objects.all(), + Map.objects.all(), + Document.objects.all(), + Tag.objects.all(), + TaggedItem.objects.all(), + ] + ], + [], + ), + indent=2, + use_natural_keys=True, + ) + if path is None: + parent, _ = os.path.split(__file__) + path = os.path.join(parent, "fixtures", "search_testdata.json") + with open(path, "w") as f: + f.write(result) + + +def create_single_dataset(name, keywords=None, owner=None, group=None, **kwargs): + admin, created = get_user_model().objects.get_or_create(username="admin") + if created: + admin.is_superuser = True + admin.first_name = "admin" + admin.set_password("admin") + admin.save() + test_datetime = datetime.strptime("2020-01-01", "%Y-%m-%d") + ll = (name, "lorem ipsum", name, f"geonode:{name}", [0, 22, 0, 22], test_datetime, ("populartag",), "farming") + title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws, category = ll + try: + dataset, _ = Dataset.objects.get_or_create( + alternate=alternate, + defaults=dict( + title=title, + abstract=abstract, + name=name, + bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + srid="EPSG:4326", + uuid=str(uuid4()), + owner=owner or admin, + temporal_extent_start=test_datetime, + temporal_extent_end=test_datetime, + date=start, + subtype="vector", + resource_type="dataset", + typename=f"geonode:{title}", + group=group, + **kwargs, + ), + ) + + if isinstance(keywords, list): + dataset = add_keywords_to_resource(dataset, keywords) + + dataset.set_default_permissions(owner=owner or admin) + dataset.clear_dirty_state() + dataset.set_processing_state(enumerations.STATE_PROCESSED) + return dataset + except IntegrityError: + return Dataset.objects.get(alternate=alternate) + + +def create_single_map(name, owner=None, **kwargs): + admin, created = get_user_model().objects.get_or_create(username="admin") + if created: + admin.is_superuser = True + admin.first_name = "admin" + admin.set_password("admin") + admin.save() + test_datetime = datetime.strptime("2020-01-01", "%Y-%m-%d") + ll = (name, "lorem ipsum", name, f"{name}", [0, 22, 0, 22], test_datetime, ("populartag",)) + title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws = ll + m, _ = Map.objects.get_or_create( + title=title, + defaults=dict( + uuid=str(uuid4()), + abstract=abstract, + owner=owner or admin, + bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + srid="EPSG:4326", + resource_type="map", + **kwargs, + ), + ) + m.set_default_permissions(owner=owner or admin) + m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) + return m + + +def create_single_doc(name, owner=None, **kwargs): + admin, created = get_user_model().objects.get_or_create(username="admin") + if created: + admin.is_superuser = True + admin.first_name = "admin" + admin.set_password("admin") + admin.save() + test_datetime = datetime.strptime("2020-01-01", "%Y-%m-%d") + dd = (name, "lorem ipsum", name, f"{name}", [0, 22, 0, 22], test_datetime, ("populartag",)) + title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws = dd + logger.debug(f"[SetUp] Add document {title}") + m, _ = Document.objects.get_or_create( + title=title, + defaults=dict( + uuid=str(uuid4()), + abstract=abstract, + owner=owner or admin, + bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + srid="EPSG:4326", + resource_type="document", + **kwargs, + ), + ) + _, _ = create_asset_and_link(m, m.owner, dfile) + m.set_default_permissions(owner=owner or admin) + m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) + return m + + +def create_single_geoapp(name, resource_type="geostory", owner=None, **kwargs): + admin, created = get_user_model().objects.get_or_create(username="admin") + if created: + admin.is_superuser = True + admin.first_name = "admin" + admin.set_password("admin") + admin.save() + test_datetime = datetime.strptime("2020-01-01", "%Y-%m-%d") + dd = (name, "lorem ipsum", name, f"{name}", [0, 22, 0, 22], test_datetime, ("populartag",)) + title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws = dd + logger.debug(f"[SetUp] Add geoapp {title}") + m, _ = GeoApp.objects.get_or_create( + title=title, + defaults=dict( + uuid=str(uuid4()), + abstract=abstract, + owner=owner or admin, + resource_type=resource_type, + bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), + srid="EPSG:4326", + **kwargs, + ), + ) + _, _ = create_asset_and_link(m, m.owner, dfile) + m.set_default_permissions(owner=owner or admin) + m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) + return m + + +def add_keywords_to_resource(resource, keywords): + for keyword in keywords: + resource.keywords.add(keyword) + resource.save() + return resource + + +if __name__ == "__main__": + create_models() + dump_models() diff --git a/geonode/base/templates/admin/thesauri/change_list.html b/geonode/base/templates/admin/thesauri/change_list.html new file mode 100644 index 0000000..030fb1b --- /dev/null +++ b/geonode/base/templates/admin/thesauri/change_list.html @@ -0,0 +1,10 @@ +{% extends 'admin/change_list.html' %} + +
+ +{{ block.super }} diff --git a/geonode/base/templates/admin/thesauri/upload_form.html b/geonode/base/templates/admin/thesauri/upload_form.html new file mode 100644 index 0000000..4144d80 --- /dev/null +++ b/geonode/base/templates/admin/thesauri/upload_form.html @@ -0,0 +1,13 @@ +{% extends 'admin/base.html' %} + +{% block content %} +
+
+ {{ form.as_p }} + {% csrf_token %} + +
+
+
+ +{% endblock %}{{ block.super }} diff --git a/geonode/base/templates/base/_resourcebase_contact_snippet.html b/geonode/base/templates/base/_resourcebase_contact_snippet.html new file mode 100644 index 0000000..ec07e1a --- /dev/null +++ b/geonode/base/templates/base/_resourcebase_contact_snippet.html @@ -0,0 +1,23 @@ +{% load i18n %} +
  • +

    {% trans "About" %}

    + {% if resource.owner == resource.poc and resource.owner == resource.metadata_author %} +

    {% trans "Responsible, Point of Contact, Metadata Author" %}

    + {% with resource.owner as profile %} + {% include "people/_profile_about_item.html" %} + {% endwith %} + {% else %} +

    {% trans "Responsible" %}

    + {% with resource.owner as profile %} + {% include "people/_profile_about_item.html" %} + {% endwith %} +

    {% trans "Point of Contact" %}

    + {% with resource.poc as profile %} + {% include "people/_profile_about_item.html" %} + {% endwith %} +

    {% trans "Metadata Author" %}

    + {% with resource.metadata_author as profile %} + {% include "people/_profile_about_item.html" %} + {% endwith %} + {% endif %} +
    • \ No newline at end of file diff --git a/geonode/base/templates/base/_resourcebase_info_panel.html b/geonode/base/templates/base/_resourcebase_info_panel.html new file mode 100644 index 0000000..4d9c5a8 --- /dev/null +++ b/geonode/base/templates/base/_resourcebase_info_panel.html @@ -0,0 +1,214 @@ +{% load i18n %} +{% block content %} +{% load thesaurus %} + +
    + {% if resource.title %} +
    {% trans "Title" %}
    +
    {{ resource.title|truncatechars:80 }}
    + {% endif %} + + {% if resource.srid and SRID_DETAIL == 'above' %} +
    {% trans "SRID" %}
    +
    {{ resource.srid }}
    + {% endif %} + + {% if LICENSES_ENABLED and LICENSES_DETAIL == 'above' and resource.license %} +
    {% trans "License" %}
    +
    {{ resource.license.name_long }}
    + {% endif %} +
    + {% for bullet in resource.license.description_bullets %} + {{ bullet }}

    + {% endfor %} + {% if resource.license.url %} + + {% trans "For more info see" %} {{ resource.license.url }}. + {% endif %} +
    + + {% if resource.abstract %} +
    {% trans "Abstract" %}
    +
    {{ resource.abstract|safe }}
    + {% endif %} + + {% if resource.date %} + {% if resource.date_type == 'creation' %} +
    {% trans "Creation Date" %}
    + {% elif resource.date_type == 'publication' %} +
    {% trans "Publication Date" %}
    + {% elif resource.date_type == 'revision' %} +
    {% trans "Revision Date" %}
    + {% else %} +
    {% trans resource.date_type|title %} {% trans "Date" %}
    + {% endif %} + {% if resource.date_type == 'creation' %} +
    {{ resource.date }}
    + {% elif resource.date_type == 'publication' %} + + {% else %} +
    {{ resource.date }}
    + {% endif %} + {% endif %} + + {% if resource.display_type %} +
    {% trans "Type" %}
    +
    {{ resource.display_type }}
    + {% endif %} + + {% if resource.keywords.count > 0 or resource.tkeywords.count > 0 %} +
    {% trans "Keywords" %}
    +
    +
      + {% for keyword in resource.keywords.all %} +
    • + + {{ keyword.name| safe}} + +
      • + {% endfor %} + {% for tid in resource.tkeywords.all|get_unique_thesaurus_set %} +
    • {{ tid|get_thesaurus_translation_by_id }} + {% for tkeywords in resource.tkeywords.all %} +
        + {% if tid == tkeywords.thesaurus.id %} +
      • {{tkeywords|get_thesaurus_localized_label}}
        • + {% endif %} +
      + {% endfor %} +
      • + {% endfor %} +
    +
    + {% endif %} + + {% if resource.category %} +
    {% trans "Category" %}
    +
    {% trans resource.category.gn_description %} {% if resource.category.description %}{% endif %}
    + {% if resource.category.description %} +
    + {% trans resource.category.description %} +
    + {% endif %} + {% endif %} + + {% if resource.regions.all %} +
    {% trans "Regions" %}
    +
    + {% for region in resource.regions.all %} + + {{ region.name }} + + {% if not forloop.last %},{% endif %} + {% endfor %} +
    + {% endif %} + + {% if resource.owner %} +
    {% trans "Responsible" %}
    +
    + {% endif %} + + {% if resource.poc %} +
    {% trans "Point of Contact" %}
    + {% for user in resource.poc %} +
    {{ user.username }}
    + {% endfor%} + {% endif %} + + {% if resource.group %} +
    {% trans "Group" %}
    +
    {{ group }}
    + {% endif %} + + {% if resource.doi %} +
    {% trans "DOI" %}
    +
    {{ resource.doi }}
    + {% endif %} + + {% if resource.attribution %} +
    {% trans "Attribution" %}
    +
    {{ resource.attribution }}
    + {% endif %} + +
    + +
    +
    {% trans "More info" %}
    +
    -
    +
    + +
    +
    + + {% if resource.srid and SRID_DETAIL == 'below' %} +
    {% trans "SRID" %}
    +
    {{ resource.srid }}
    + {% endif %} + + {% if LICENSES_ENABLED and LICENSES_DETAIL == 'below' and resource.license %} +
    {% trans "License" %}
    +
    {{ resource.license.name_long }}
    +
    + {% for bullet in resource.license.description_bullets %} + {{ bullet }}

    + {% endfor %} + {% if resource.license.url %} + + {% trans "For more info see" %} {{ resource.license.url }}. + {% endif %} +
    + {% endif %} + + {% if resource.maintenance_frequency %} +
    {% trans "Maintenance Frequency" %}
    +
    {{ resource.maintenance_frequency_title|safe }}
    + {% endif %} + + {% if resource.restriction_code_type or resource.constraints_other %} +
    {% trans "Restrictions" %}
    +
    {% if resource.constraints_other %} + {{ resource.constraints_other|safe }} + {% else %} + {{ resource.restriction_code_type|safe }} + {% endif %}
    + {% endif %} + + {% if resource.edition %} +
    {% trans "Edition" %}
    +
    {{ resource.edition }}
    + {% endif %} + + {% if resource.purpose %} +
    {% trans "Purpose" %}
    +
    {{ resource.purpose|safe }}
    + {% endif %} + + {% if resource.language %} +
    {% trans "Language" %}
    +
    {{ resource.language_title|safe }}
    + {% endif %} + + {% if resource.temporal_extent_start and resource.temporal_extent_end %} +
    {% trans "Temporal Extent" %}
    +
    {{ resource.temporal_extent_start }} - {{ resource.temporal_extent_end }}
    + {% endif %} + + {% if resource.data_quality_statement %} +
    {% trans "Data Quality" %}
    +
    {{ resource.data_quality_statement|safe }}
    + {% endif %} + + {% if resource.supplemental_information %} +
    {% trans "Supplemental Information" %}
    +
    {{ resource.supplemental_information|safe }}
    + {% endif %} + + {% if resource.spatial_representation_type %} +
    {% trans "Spatial Representation Type" %}
    +
    {{ resource.spatial_representation_type|safe }}
    + {% endif %} + +
    + +
    + + {% endblock %} diff --git a/geonode/base/templates/base/_resourcebase_opengraph.html b/geonode/base/templates/base/_resourcebase_opengraph.html new file mode 100644 index 0000000..a45b76a --- /dev/null +++ b/geonode/base/templates/base/_resourcebase_opengraph.html @@ -0,0 +1,7 @@ + + + + + + + diff --git a/geonode/base/templates/base/_resourcebase_snippet.html b/geonode/base/templates/base/_resourcebase_snippet.html new file mode 100644 index 0000000..fb394b0 --- /dev/null +++ b/geonode/base/templates/base/_resourcebase_snippet.html @@ -0,0 +1,92 @@ +{% load i18n %} +{% verbatim %} + +
    +
    + {% endverbatim %} +

    {% trans "No content created yet." %}

    + {% verbatim %} +
    +
    +
    +
    +
    + + + +
    +
    +
    +
    +

    + {{ item.category__gn_description }}
    + {{ item.group_name }}
     + + {% endverbatim %} + {% trans "Temporal Serie" %}
    + {% verbatim %} +     +

    +

    + + + + + + + {{ item.title }} +

    +
    + {% endverbatim %} + + {% if facet_type == 'datasets' %} +
    +

    + +

    +
    + {% endif %} +
    + + {% trans "Service is" %} {% trans "online" %} + {% trans "Service is" %} {% trans "offline" %} + +
    {% trans "SECURITY NOT YET SYNCHRONIZED" %} + {% verbatim %}{% endverbatim %}{% trans "Sync permissions immediately" %} +
    +
    {% trans "PENDING APPROVAL" %}
    +
    {% trans "UNPUBLISHED" %}
    + + {% verbatim %} +
    +

    {{ item.abstract.length > 300 ? '...' : ''}}

    +
    +
    + +
    +
    +
    +
    +
    +
    +
    +{% endverbatim %} diff --git a/geonode/base/templates/base/_resourcebase_twittercard.html b/geonode/base/templates/base/_resourcebase_twittercard.html new file mode 100644 index 0000000..7370b15 --- /dev/null +++ b/geonode/base/templates/base/_resourcebase_twittercard.html @@ -0,0 +1,7 @@ + + + + + + + diff --git a/geonode/base/templates/base/base_embed.html b/geonode/base/templates/base/base_embed.html new file mode 100644 index 0000000..fc2f477 --- /dev/null +++ b/geonode/base/templates/base/base_embed.html @@ -0,0 +1,14 @@ +{% load i18n %} +{% load base_tags %} +{% load client_lib_tags %} + +{% block head %} + + + + {% get_resourcebase_embed %} +{% endblock %} \ No newline at end of file diff --git a/geonode/base/templates/base/base_metadata.html b/geonode/base/templates/base/base_metadata.html new file mode 100644 index 0000000..202ff74 --- /dev/null +++ b/geonode/base/templates/base/base_metadata.html @@ -0,0 +1,87 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load bootstrap_tags %} +{% load base_tags %} +{% load guardian_tags %} +{% load floppyforms %} + +{% block title %}{{ resource.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock body_class %} + +{% block body_outer %} + +
    +

    {% trans "Metadata" %} {% blocktrans with resource.title as map_title %}for {{ map_title }}{% endblocktrans %}

    +
    + {% trans "Completeness" %} +
    + + +
    +
    +
    + 70% +
    +
    +
    +
    + +
    + {% if resource.metadata_uploaded %} +
    {% blocktrans %}Note: this resource's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}
    + {% endif %} + + {% if resourcebase_form.errors or category_form.errors or tkeywords_form.errors %} +
    {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %} +
      + {% for field in resourcebase_form %} + {% if field.errors %} +
    • {{ field.label }}
      • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
    • {{ category_form.errors.as_ul }}
      • + {% endif %} + {% if tkeywords_form.errors %} +
    • {{ tkeywords_form.errors.as_ul }}
      • + {% endif %} +
    +
    + {% endif %} + + {% csrf_token %} +
    + {% form resourcebase_form using panel_template %} + {# resourcebase_form|as_bootstrap #} +
    + +
    +
    + + + +
    + + + >" %}"/> +
    +
    +
    +
    + + + +{{ block.super }} +{% endblock body_outer %} diff --git a/geonode/base/templates/base/base_metadata_advanced.html b/geonode/base/templates/base/base_metadata_advanced.html new file mode 100644 index 0000000..8eba9be --- /dev/null +++ b/geonode/base/templates/base/base_metadata_advanced.html @@ -0,0 +1,134 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load static %} +{% load base_tags %} +{% load bootstrap_tags %} +{% load guardian_tags %} +{% load client_lib_tags %} +{% block title %}{{ resourcebase.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock %} + +{% block body_outer %} + +{{ block.super }} + + + + + + + + + + +
    +

    {% trans "Edit Metadata" %}

    +
    +
    +
    +

    + {% blocktrans with resourcebase.title as map_title %} + Editing details for {{ map_title }} + {% endblocktrans %} +

    + +
    + {% if resourcebase.metadata_uploaded %} +
    {% blocktrans %}Note: this resourcebase's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}
    + {% endif %} + + {% if resourcebase_form.errors or category_form.errors %} +
    {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %} +
      + {% for field in resourcebase_form %} + {% if field.errors %} +
    • {{ field.label }}
      • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
    • {{ category_form.errors.as_ul }}
      • + {% endif %} +
    +
    + {% endif %} +
    + +
    + {% csrf_token %} + +
    + {% block resourcebase_fields %} + {% for field in resourcebase_form %} + {% if field.name != 'use_featureinfo_custom_template' and field.name != 'featureinfo_custom_template' and field.name not in ADVANCED_EDIT_EXCLUDE_FIELD %} + {% if field.name == 'featured' and not user.is_superuser %} + {% else %} +
    +
    + + {{ field }} +
    +
    + {% endif %} + {% endif %} + {% endfor %} + {% endblock resourcebase_fields %} + + + {% block thesauri %} + {% if THESAURI_FILTERS %} + {% for field in tkeywords_form %} +
    +

    + + {{ field }} +

    +
    + {% endfor %} + {% endif %} + {% endblock thesauri %} +
    +
    +
    + +
    + {% autoescape off %} + {% for choice in category_form.category_choice_field.field.choices %} +
    + +
    + {% endfor %} + {% endautoescape %} +
    +
    + +
    + + + +
    + +
    +
    +
    +
    +
    +
    +{% endblock %} diff --git a/geonode/base/templates/base/base_metadata_detail.html b/geonode/base/templates/base/base_metadata_detail.html new file mode 100644 index 0000000..a1c62b4 --- /dev/null +++ b/geonode/base/templates/base/base_metadata_detail.html @@ -0,0 +1,6 @@ +{% extends "metadata_detail.html" %} +{% load i18n %} +{% block metaget_absolute_url %} +
    {% trans "Metadata Page" %}
    +
    {% url "resourcebase_metadata_detail" resource.id %}
    +{% endblock metaget_absolute_url %} \ No newline at end of file diff --git a/geonode/base/templates/base/base_panels.html b/geonode/base/templates/base/base_panels.html new file mode 100644 index 0000000..e82cf60 --- /dev/null +++ b/geonode/base/templates/base/base_panels.html @@ -0,0 +1,607 @@ +{% load i18n %} +{% load static %} +{% load floppyforms %} +{% load contact_roles %} + + + + + + + + + + + + + + + + + + +{% block body_outer %} + + +
    +
    +
    + +
    + {% trans "Mandatory" %} +
    +
    + {% trans "Mandatory" %} +
    +
    + {% trans "Optional" %} +
    +
    + +
    + +
    +
    + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + + +
    +
    + {% block resourcebase_title %} + + + {{ resourcebase_form.title }} + {% endblock %} +
    + + + {{ resourcebase_form.abstract }} +
    +
    +
    +
    + + + {{ resourcebase_form.date_type }} +
    +
    + + + {{ resourcebase_form.date }} +
    + {% block resourcebase_category %} +
    + + +
    + {% endblock resourcebase_category %} +
    + + +
    +
    + + {{ resourcebase_form.keywords }} +
    + {% if THESAURI_FILTERS %} +
    + {{tkeywords_form.as_p}} +
    + {% endif %} +
    +
    +
    +
    +
    +
    +
    +
    + +
    +
    +
    + +
    +
    +
    + {% block resourcebase_attributes %} +
    +
    + + + {{ resourcebase_form.language }} +
    +
    + + + {{ resourcebase_form.license }} +
    +
    + + {{ resourcebase_form.attribution }} +
    +
    + {% endblock resourcebase_attributes %} +
    +
    + + {{ resourcebase_form.regions }} +
    +
    + + + {{ resourcebase_form.data_quality_statement }} +
    +
    +
    +
    + + + {{ resourcebase_form.restriction_code_type }} +
    +
    + + + {{ resourcebase_form.constraints_other }} +
    +
    +
    +
    + +
    +
    +
    +
    +
    +
    + +
    +
    +
    +
    +

    {% trans "Other, Optional, Metadata" %}

    +
    + + + {{ resourcebase_form.edition }} +
    +
    + + {{ resourcebase_form.doi }} +
    +
    + + + {{ resourcebase_form.purpose }} +
    +
    + + + {{ resourcebase_form.supplemental_information }} +
    +
    +
    + {% block resourcebase_temporal_extent_start %} +
    +
    + + + {{ resourcebase_form.temporal_extent_start }} +
    +
    + {% endblock resourcebase_temporal_extent_start %} + {% block resourcebase_temporal_extent_end %} +
    +
    + + + {{ resourcebase_form.temporal_extent_end }} +
    +
    + {% endblock resourcebase_temporal_extent_end %} + {% block maintenance_block %} +
    +
    + + + {{ resourcebase_form.maintenance_frequency }} +
    +
    + + + {{ resourcebase_form.spatial_representation_type }} +
    + {% block resourcebase_extra_metadata %} +
    + + {{ resourcebase_form.extra_metadata }} +
    + {% endblock resourcebase_extra_metadata %} + {% block resourcebase_linked_resources %} +
    + + {{ resourcebase_form.linked_resources }} +
    + {% endblock resourcebase_linked_resources %} + +
    + {% endblock maintenance_block %} +
    +
    + {% block resourcebase_poc %} +
    +
    {% trans "Responsible Parties" %}
    +
    + + {{ resourcebase_form.poc }} +
    +
    + {% endblock %} +
    +
    {% trans "Responsible and Permissions" %}
    + {% block resourcebase_owner %} +
    +
    + + {{ resourcebase_form.owner }} +
    + {% endblock resourcebase_owner %} +
    +
    + {% trans "toggle more Contact Roles" %} + {% block resourcebase_more_contact_roles %} +
    +
    {% trans "more metadata contact roles" %}
    + {% for contact_role in UI_ROLES_IN_TOGGLE_VIEW %} + {% getattribute resourcebase_form contact_role as cr %} +
    +
    + + {{ cr}} +
    +
    + {% endfor %} +
    +
    + {% endblock resourcebase_more_contact_roles %} +
    + +
    +
    +
    + {% block extra_metadata_content %} + {% endblock %} +
    +
    + +
    +
    +
    +
    +
    +
    {% trans "Publishing" %}
    +
    +
    + + {{ resourcebase_form.metadata_uploaded_preserve }} +
    +
    + + {{ resourcebase_form.is_approved }} +
    +
    + + {{ resourcebase_form.is_published }} +
    + {% if user.is_superuser %} +
    + + {{ resourcebase_form.featured }} +
    + {% endif %} +
    + + {{ resourcebase_form.advertised }} +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    {% trans "Other Settings" %}
    +
    +
    + + {{ resourcebase_form.id }} +
    +
    +
    +
    +
    +
    +
    +
    +
    +{% endblock %} diff --git a/geonode/base/templates/base/batch_edit.html b/geonode/base/templates/base/batch_edit.html new file mode 100644 index 0000000..1ffb23f --- /dev/null +++ b/geonode/base/templates/base/batch_edit.html @@ -0,0 +1,22 @@ +{% extends "geonode_base.html" %} +{% load i18n %} +{% load static %} +{% load bootstrap_tags %} + +{% block title %} {% trans "Batch Edit" %} - {{ block.super }} {% endblock %} + +{% block body_class %}batch edit{% endblock %} + +{% block body %} +
    +

    {% trans "Batch Edit" %}

    +
    +
    + {% csrf_token %} + {{ form|as_bootstrap }} +
    + + +
    +
    +{% endblock %} diff --git a/geonode/base/templates/base/iso_categories.html b/geonode/base/templates/base/iso_categories.html new file mode 100644 index 0000000..aca0d04 --- /dev/null +++ b/geonode/base/templates/base/iso_categories.html @@ -0,0 +1,22 @@ +{% load i18n %} +{% if iso_formats %} +
    +
    +
    +

    {% trans "Discover the available datasets." %}

    +
    + {% for iso_format in iso_formats %} +
    +
    + {{ iso_format.count }} + +

    {% trans iso_format.category__gn_description %}

    +
    +
    + {% endfor %} +
    +
    +
    +
    +{% endif %} \ No newline at end of file diff --git a/geonode/base/templates/base/maintenance.html b/geonode/base/templates/base/maintenance.html new file mode 100644 index 0000000..ee91c9c --- /dev/null +++ b/geonode/base/templates/base/maintenance.html @@ -0,0 +1,45 @@ +{% load static %} + + + + + + + + +
    +
    + +

    We're doing critical maintenance of the site, which we do from time to time to bring out upgrades and new functionality.

    Please come back and try again soon

    +
    +
    + + + diff --git a/geonode/base/templates/base/menu.html b/geonode/base/templates/base/menu.html new file mode 100644 index 0000000..e818893 --- /dev/null +++ b/geonode/base/templates/base/menu.html @@ -0,0 +1,84 @@ + + +{% for menu, menu_items in menus.items %} + + {% if menu_items|length > 0 %} + +
  • + + + {{ menu.title }} + {% if menu_items|length > 0 %} + + {% endif %} + + + +
    • + + {% endif %} + +{% endfor %} +{% block content_reposition_script %} + +{% endblock %} diff --git a/geonode/base/templates/base/read_only_violation.html b/geonode/base/templates/base/read_only_violation.html new file mode 100644 index 0000000..e869cf4 --- /dev/null +++ b/geonode/base/templates/base/read_only_violation.html @@ -0,0 +1,46 @@ +{% load static %} + + + + + + + + +
    +
    + +

    Your instance is currently in Read-Only mode.

    Performing any changes is disallowed on this instace.

    +
    +
    + + + diff --git a/geonode/base/templates/base/resourcebase_info_panel.html b/geonode/base/templates/base/resourcebase_info_panel.html new file mode 100644 index 0000000..13a0164 --- /dev/null +++ b/geonode/base/templates/base/resourcebase_info_panel.html @@ -0,0 +1,10 @@ +{% extends "base/_resourcebase_info_panel.html" %} +{% load i18n %} + +
    + {% block article_content %} + {% block content %} + {{ block.super }} + {% endblock %} + {% endblock %} +
    \ No newline at end of file diff --git a/geonode/base/templates/base/thumbnail_upload.html b/geonode/base/templates/base/thumbnail_upload.html new file mode 100644 index 0000000..177494a --- /dev/null +++ b/geonode/base/templates/base/thumbnail_upload.html @@ -0,0 +1,47 @@ +{% extends "geonode_base.html" %} +{% load i18n %} + + +{% block body_class %}Upload a Thumbnail{% endblock %} + +{% block body_outer %} +{% if error %} +
    {{ error }}
    +{% endif %} + +
    +

    {% trans "Upload a Thumbnail" %}

    +
    + +
    + +
    + + + {% if resource.thumbnail_url %} +
    Current thumbnail
    + +
    + {% csrf_token %} + +
    + {% endif %} + + +
    + {% csrf_token %} + {{ form.img }} + +
    + +
    + +
    +
    {% trans "Auto generated thumbnail" %}
    + +
    + +
    + +{% endblock %} diff --git a/geonode/base/templates/base/user_and_group_permissions.html b/geonode/base/templates/base/user_and_group_permissions.html new file mode 100644 index 0000000..cfe3f0f --- /dev/null +++ b/geonode/base/templates/base/user_and_group_permissions.html @@ -0,0 +1,49 @@ +{% extends "geonode_base.html" %} +{% load i18n %} +{% load static %} +{% load bootstrap_tags %} + +{% block title %} {% trans "Batch Edit" %} - {{ block.super }} {% endblock %} + +{% block body_class %}batch edit{% endblock %} + +{% block body %} + + + + + + + + + + + + +
    +

    {% trans "Batch Edit" %}

    +
    +
    + {% csrf_token %} +
    + + {{ form.layers }} +
    +
    +
    + + {{ form.permission_type }} +
    +
    + + {{ form.mode }} +
    +
    + {{ form.ids }} +
    +
    + + +
    +
    +{% endblock %} diff --git a/geonode/base/templates/geonode_base.html b/geonode/base/templates/geonode_base.html new file mode 100644 index 0000000..63913c1 --- /dev/null +++ b/geonode/base/templates/geonode_base.html @@ -0,0 +1 @@ +{% extends "base.html" %} \ No newline at end of file diff --git a/geonode/base/templatetags/__init__.py b/geonode/base/templatetags/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/base/templatetags/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/base/templatetags/base_tags.py b/geonode/base/templatetags/base_tags.py new file mode 100644 index 0000000..d9bf63f --- /dev/null +++ b/geonode/base/templatetags/base_tags.py @@ -0,0 +1,423 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django import template +from django.db.models import Q +from django.conf import settings +from django.db.models import Count +from django.utils.translation import gettext_lazy +from django.contrib.auth import get_user_model +from django.utils.translation import gettext_lazy as _ + +from guardian.shortcuts import get_objects_for_user + +from geonode.maps.models import Map +from geonode.layers.models import Dataset +from geonode.base.models import ResourceBase +from geonode.documents.models import Document +from geonode.groups.models import GroupProfile +from geonode.base.bbox_utils import filter_bbox +from geonode.base.models import HierarchicalKeyword, Menu, MenuItem +from geonode.security.utils import get_visible_resources +from collections import OrderedDict + +register = template.Library() + +FACETS = { + "raster": _("Raster Dataset"), + "vector": _("Vector Dataset"), + "vector_time": _("Vector Temporal Serie"), + "remote": _("Remote Dataset"), + "wms": _("WMS Cascade Dataset"), +} + + +@register.filter(name="template_trans") +def template_trans(text): + try: + return gettext_lazy(text) + except Exception: + return text + + +@register.filter(name="get_item") +def get_item(dictionary, key): + """Get a element for a dict by name""" + return dictionary.get(key) + + +@register.simple_tag(takes_context=True) +def facets(context): + request = context["request"] + title_filter = request.GET.get("title__icontains", "") + abstract_filter = request.GET.get("abstract__icontains", "") + purpose_filter = request.GET.get("purpose__icontains", "") + extent_filter = request.GET.get("extent", None) + keywords_filter = request.GET.getlist("keywords__slug__in", None) + category_filter = request.GET.getlist("category__identifier__in", None) + regions_filter = request.GET.getlist("regions__name__in", None) + owner_filter = request.GET.getlist("owner__username__in", None) + date_gte_filter = request.GET.get("date__gte", None) + date_lte_filter = request.GET.get("date__lte", None) + date_range_filter = request.GET.get("date__range", None) + + facet_type = context.get("facet_type", "all") + + if not settings.SKIP_PERMS_FILTER: + authorized = [] + try: + authorized = get_objects_for_user(request.user, "base.view_resourcebase").values("id") + except Exception: + pass + + if facet_type == "geoapps": + facets = {} + + from django.apps import apps + + for label, app in apps.app_configs.items(): + if hasattr(app, "type") and app.type == "GEONODE_APP": + if hasattr(app, "default_model"): + geoapps = get_visible_resources( + apps.get_model(label, app.default_model).objects.all(), + request.user if request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + if category_filter: + geoapps = geoapps.filter(category__identifier__in=category_filter) + if regions_filter: + geoapps = geoapps.filter(regions__name__in=regions_filter) + if owner_filter: + geoapps = geoapps.filter(owner__username__in=owner_filter) + if date_gte_filter: + geoapps = geoapps.filter(date__gte=date_gte_filter) + if date_lte_filter: + geoapps = geoapps.filter(date__lte=date_lte_filter) + if date_range_filter: + geoapps = geoapps.filter(date__range=date_range_filter.split(",")) + + if extent_filter: + geoapps = filter_bbox(geoapps, extent_filter) + + if keywords_filter: + treeqs = HierarchicalKeyword.objects.none() + for keyword in keywords_filter: + try: + kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) + for kw in kws: + treeqs = treeqs | HierarchicalKeyword.get_tree(kw) + except Exception: + # Ignore keywords not actually used? + pass + + geoapps = geoapps.filter(Q(keywords__in=treeqs)) + + if not settings.SKIP_PERMS_FILTER: + geoapps = geoapps.filter(id__in=authorized) + + facets[app.default_model] = geoapps.count() + return facets + elif facet_type == "documents": + documents = Document.objects.filter(title__icontains=title_filter) + if category_filter: + documents = documents.filter(category__identifier__in=category_filter) + if regions_filter: + documents = documents.filter(regions__name__in=regions_filter) + if owner_filter: + documents = documents.filter(owner__username__in=owner_filter) + if date_gte_filter: + documents = documents.filter(date__gte=date_gte_filter) + if date_lte_filter: + documents = documents.filter(date__lte=date_lte_filter) + if date_range_filter: + documents = documents.filter(date__range=date_range_filter.split(",")) + + documents = get_visible_resources( + documents, + request.user if request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + if keywords_filter: + treeqs = HierarchicalKeyword.objects.none() + for keyword in keywords_filter: + try: + kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) + for kw in kws: + treeqs = treeqs | HierarchicalKeyword.get_tree(kw) + except Exception: + # Ignore keywords not actually used? + pass + + documents = documents.filter(Q(keywords__in=treeqs)) + + if not settings.SKIP_PERMS_FILTER: + documents = documents.filter(id__in=authorized) + + counts = documents.values("subtype").annotate(count=Count("subtype")) + facets = {count["subtype"]: count["count"] for count in counts} + + return facets + else: + layers = Dataset.objects.filter( + Q(title__icontains=title_filter) + | Q(abstract__icontains=abstract_filter) + | Q(purpose__icontains=purpose_filter) + ) + if category_filter: + layers = layers.filter(category__identifier__in=category_filter) + if regions_filter: + layers = layers.filter(regions__name__in=regions_filter) + if owner_filter: + layers = layers.filter(owner__username__in=owner_filter) + if date_gte_filter: + layers = layers.filter(date__gte=date_gte_filter) + if date_lte_filter: + layers = layers.filter(date__lte=date_lte_filter) + if date_range_filter: + layers = layers.filter(date__range=date_range_filter.split(",")) + + layers = get_visible_resources( + layers, + request.user if request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + if extent_filter: + layers = filter_bbox(layers, extent_filter) + + if keywords_filter: + treeqs = HierarchicalKeyword.objects.none() + for keyword in keywords_filter: + try: + kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) + for kw in kws: + treeqs = treeqs | HierarchicalKeyword.get_tree(kw) + except Exception: + # Ignore keywords not actually used? + pass + + layers = layers.filter(Q(keywords__in=treeqs)) + + if not settings.SKIP_PERMS_FILTER: + layers = layers.filter(id__in=authorized) + + counts = layers.values("subtype").annotate(count=Count("subtype")) + + counts_array = [] + try: + for count in counts: + counts_array.append((count["subtype"], count["count"])) + except Exception: + pass + + count_dict = dict(counts_array) + + vector_time_series = ( + layers.exclude(has_time=False).filter(subtype="vector").values("subtype").annotate(count=Count("subtype")) + ) + + if vector_time_series: + count_dict["vectorTimeSeries"] = vector_time_series[0]["count"] + + facets = { + "raster": count_dict.get("raster", 0), + "vector": count_dict.get("vector", 0), + "vector_time": count_dict.get("vectorTimeSeries", 0), + "remote": count_dict.get("remote", 0), + "wms": count_dict.get("wmsStore", 0), + } + + # Break early if only_datasets is set. + if facet_type == "datasets": + return facets + + maps = Map.objects.filter(title__icontains=title_filter) + documents = Document.objects.filter(title__icontains=title_filter) + + if category_filter: + maps = maps.filter(category__identifier__in=category_filter) + documents = documents.filter(category__identifier__in=category_filter) + if regions_filter: + maps = maps.filter(regions__name__in=regions_filter) + documents = documents.filter(regions__name__in=regions_filter) + if owner_filter: + maps = maps.filter(owner__username__in=owner_filter) + documents = documents.filter(owner__username__in=owner_filter) + if date_gte_filter: + maps = maps.filter(date__gte=date_gte_filter) + documents = documents.filter(date__gte=date_gte_filter) + if date_lte_filter: + maps = maps.filter(date__lte=date_lte_filter) + documents = documents.filter(date__lte=date_lte_filter) + if date_range_filter: + maps = maps.filter(date__range=date_range_filter.split(",")) + documents = documents.filter(date__range=date_range_filter.split(",")) + + maps = get_visible_resources( + maps, + request.user if request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + documents = get_visible_resources( + documents, + request.user if request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + if extent_filter: + documents = filter_bbox(documents, extent_filter) + + if keywords_filter: + treeqs = HierarchicalKeyword.objects.none() + for keyword in keywords_filter: + try: + kws = HierarchicalKeyword.objects.filter(name__iexact=keyword) + for kw in kws: + treeqs = treeqs | HierarchicalKeyword.get_tree(kw) + except Exception: + # Ignore keywords not actually used? + pass + + maps = maps.filter(Q(keywords__in=treeqs)) + documents = documents.filter(Q(keywords__in=treeqs)) + + if not settings.SKIP_PERMS_FILTER: + maps = maps.filter(id__in=authorized) + documents = documents.filter(id__in=authorized) + + facets["map"] = maps.count() + facets["document"] = documents.count() + + if facet_type == "home": + facets["user"] = get_user_model().objects.exclude(username="AnonymousUser").count() + + facets["group"] = GroupProfile.objects.exclude(access="private").count() + + facets["dataset"] = facets["raster"] + facets["vector"] + facets["remote"] + facets["wms"] + + return facets + + +@register.simple_tag(takes_context=True) +def get_current_path(context): + request = context["request"] + return request.get_full_path() + + +@register.simple_tag(takes_context=True) +def get_context_resourcetype(context): + c_path = get_current_path(context) + resource_types = ["datasets", "maps", "geoapps", "documents", "search", "people", "groups/categories", "groups"] + for resource_type in resource_types: + if f"/{resource_type}/" in c_path: + return resource_type + return "error" + + +@register.simple_tag(takes_context=True) +def fullurl(context, url): + if not url: + return "" + r = context["request"] + return r.build_absolute_uri(url) + + +@register.simple_tag +def get_menu(placeholder_name): + menus = { + m: MenuItem.objects.filter(menu=m).order_by("order") + for m in Menu.objects.filter(placeholder__name=placeholder_name) + } + return OrderedDict(menus.items()) + + +@register.inclusion_tag(filename="base/menu.html") +def render_nav_menu(placeholder_name): + menus = {} + try: + menus = { + m: MenuItem.objects.filter(menu=m).order_by("order") + for m in Menu.objects.filter(placeholder__name=placeholder_name) + } + except Exception: + pass + + return {"menus": OrderedDict(menus.items())} + + +@register.inclusion_tag(filename="base/iso_categories.html") +def get_visibile_resources(user): + categories = ( + get_objects_for_user(user, "view_resourcebase", klass=ResourceBase, any_perm=False) + .filter(category__isnull=False) + .values("category__gn_description", "category__fa_class", "category__description", "category__identifier") + .annotate(count=Count("category")) + ) + + return {"iso_formats": categories} + + +@register.simple_tag +def display_edit_request_button(resource, user, perms): + def _has_owner_his_permissions(): + _owner_perms = set( + resource.BASE_PERMISSIONS.get("owner") + + resource.BASE_PERMISSIONS.get("read") + + resource.BASE_PERMISSIONS.get("write") + ) + + if resource.resource_type in ["dataset", "document"]: + """ + The download resource permission should be available only + if the resource is a datasets or Documents. You cant download maps + """ + _owner_perms = _owner_perms.union(set(resource.BASE_PERMISSIONS.get("download"))) + + _owner_set = _owner_perms.difference(set(perms)) + return _owner_set == set() or _owner_set == {"change_resourcebase_permissions", "publish_resourcebase"} + + if not _has_owner_his_permissions() and (user.is_superuser or resource.owner.pk == user.pk): + return True + return False + + +@register.simple_tag +def display_change_perms_button(resource, user, perms): + try: + from geonode.geoserver.helpers import ogc_server_settings + except Exception: + return False + if not getattr(ogc_server_settings, "GEONODE_SECURITY_ENABLED", False): + return False + elif user.is_superuser or "change_resourcebase_permissions" in set(perms): + return True + else: + return not getattr(settings, "ADMIN_MODERATE_UPLOADS", False) diff --git a/geonode/base/templatetags/sanitize_html.py b/geonode/base/templatetags/sanitize_html.py new file mode 100644 index 0000000..70b4822 --- /dev/null +++ b/geonode/base/templatetags/sanitize_html.py @@ -0,0 +1,36 @@ +from django import template +from django.conf import settings +import nh3 +from django.utils.encoding import force_str + +register = template.Library() + + +@register.filter(name="sanitize_html") +def sanitize_html(value): + """ + This filter can be used in Django templates to ensure that HTML content is safe to render. It leverages the + nh3 library (https://github.com/messense/nh3) to remove dangerous tags and JavaScript patterns. + + The allowed HTML tags and attributes can be + configured via the 'NH3_DEFAULT_CONFIG' setting in Django's settings.py. If this configuration is not set, a default + set of safe tags will be used. + + Usage in templates: {{ some_variable|sanitize_html|safe }} + + :param value: The HTML content to be sanitized within the template. + :return: Sanitized HTML content safe for rendering in templates. + """ + + # Tests: Ensure the value is converted to a string if it's a lazy object + value = force_str(value) + + nh3_config = getattr( + settings, + "NH3_DEFAULT_CONFIG", + { + "tags": {"b", "a", "img", "p", "ul", "li", "strong", "em", "span"}, + }, + ) + + return nh3.clean(value, **nh3_config) diff --git a/geonode/base/templatetags/thesaurus.py b/geonode/base/templatetags/thesaurus.py new file mode 100644 index 0000000..3b0ee9e --- /dev/null +++ b/geonode/base/templatetags/thesaurus.py @@ -0,0 +1,59 @@ +from django.core.exceptions import ObjectDoesNotExist +from django import template +from django.utils.translation import get_language +from geonode.base.models import Thesaurus, ThesaurusKeywordLabel, ThesaurusLabel +from geonode.base.utils import remove_country_from_languagecode + +register = template.Library() + + +@register.filter +def get_unique_thesaurus_set(thesaurus_from_keyword): + return set(thesaurus_from_keyword.values_list("thesaurus", flat=True)) + + +@register.filter +def get_thesaurus_title(thesaurus_id): + return Thesaurus.objects.get(id=thesaurus_id).title + + +@register.filter +def get_thesaurus_date(thesaurus_id): + return Thesaurus.objects.get(id=thesaurus_id).date + + +@register.filter +def get_name_translation(tidentifier): + lang = get_language() + lang = remove_country_from_languagecode(lang) + available = Thesaurus.objects.filter(identifier=tidentifier) + if not available.exists(): + raise ObjectDoesNotExist("Selected idenfifier does not exists") + lname = ( + ThesaurusLabel.objects.values_list("label", flat=True) + .filter(thesaurus__identifier=tidentifier) + .filter(lang=lang) + ) + if not lname: + return available.first().title + return lname.first() + + +@register.filter +def get_thesaurus_translation_by_id(id): + available = Thesaurus.objects.filter(id=id) + if not available.exists(): + raise ObjectDoesNotExist("Selected idenfifier does not exists") + return get_name_translation(available.first().identifier) + + +@register.filter +def get_thesaurus_localized_label(tkeyword): + lang = get_language() + lang = remove_country_from_languagecode(lang) + translation = ( + ThesaurusKeywordLabel.objects.values_list("label", flat=True).filter(keyword__id=tkeyword.id).filter(lang=lang) + ) + if not translation: + return tkeyword.alt_label + return translation.first() diff --git a/geonode/base/templatetags/user_messages.py b/geonode/base/templatetags/user_messages.py new file mode 100644 index 0000000..421e3a6 --- /dev/null +++ b/geonode/base/templatetags/user_messages.py @@ -0,0 +1,83 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from uuid import uuid4 +from django import template +from django.conf import settings +from django.db.models import Sum +from django.contrib.auth import get_user_model +from django.utils.translation import gettext_lazy as _ + +register = template.Library() + + +@register.simple_tag +def is_unread(thread, user): + if ( + thread.userthread_set.filter(user=user, unread=True).exists() + or thread.groupmemberthread_set.filter(user=user, unread=True).exists() + ): + return True + return False + + +@register.simple_tag +def random_uuid(): + return str(uuid4()) + + +@register.simple_tag +def format_senders(thread, current_user): + User = get_user_model() + users = User.objects.filter(sent_messages__thread=thread).annotate(Sum("pk")) + sender_string = "" + u_count = users.count() + if u_count < 3: + for user in users: + if user == current_user: + user_repr = _("me") + else: + user_repr = f"{user.full_name_or_nick}" + sender_string += f"{user_repr}, " + sender_string = sender_string[:-2] + elif u_count == 3: + for user in users: + if user == current_user: + user_repr = _("me") + else: + user_repr = f"{user.first_name_or_nick}" + sender_string += f"{user_repr}, " + sender_string = sender_string[:-2] + else: + first_sender = thread.first_message.sender + last_sender = thread.latest_message.sender + sender_string = f"{first_sender.first_name_or_nick} .. {last_sender.first_name_or_nick}" + return f"{sender_string}" + + +@register.simple_tag +def get_item(dictionary, key): + return dictionary.get(key, []) + + +@register.simple_tag +def show_notification(notice_type_label, current_user): + adms_notice_types = getattr(settings, "ADMINS_ONLY_NOTICE_TYPES", []) + if not current_user.is_superuser and adms_notice_types and notice_type_label in adms_notice_types: + return False + return True diff --git a/geonode/base/tests.py b/geonode/base/tests.py new file mode 100644 index 0000000..ec4c997 --- /dev/null +++ b/geonode/base/tests.py @@ -0,0 +1,1443 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import logging +import os +import requests +import importlib +from django.apps import apps +from PIL import Image +from io import BytesIO +from uuid import uuid4 +from unittest.mock import patch, Mock +from guardian.shortcuts import assign_perm + +from django.utils.module_loading import import_string +from django.db.utils import IntegrityError, OperationalError +from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings +from django.contrib.gis.geos import Polygon, GEOSGeometry +from django.template import Template, Context +from django.contrib.auth import get_user_model +from geonode.storage.manager import storage_manager +from django.test import Client, TestCase, override_settings, SimpleTestCase +from django.shortcuts import reverse +from django.utils import translation +from django.core.files import File +from django.core.management import call_command +from django.core.management.base import CommandError + +from geonode.base.populate_test_data import create_single_dataset +from geonode.maps.models import Map +from geonode.resource.utils import KeywordHandler +from geonode.thumbs import utils as thumb_utils +from geonode.base import enumerations +from geonode.layers.models import Dataset +from geonode.services.models import Service +from geonode.documents.models import Document +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.base.templatetags.base_tags import display_change_perms_button +from geonode.base.templatetags.sanitize_html import sanitize_html +from geonode.base.utils import OwnerRightsRequestViewUtils +from geonode.base.models import ( + HierarchicalKeyword, + ResourceBase, + MenuPlaceholder, + License, + Group, + Menu, + MenuItem, + Configuration, + Region, + TopicCategory, + Thesaurus, + ThesaurusKeyword, + generate_thesaurus_reference, +) +from geonode.base.middleware import ReadOnlyMiddleware, MaintenanceMiddleware +from geonode.base.templatetags.base_tags import get_visibile_resources, facets +from geonode.base.templatetags.thesaurus import ( + get_name_translation, + get_thesaurus_localized_label, + get_thesaurus_translation_by_id, + get_unique_thesaurus_set, + get_thesaurus_title, + get_thesaurus_date, +) +from geonode.base.templatetags.user_messages import show_notification +from geonode import geoserver +from geonode.decorators import on_ogc_backend +from geonode.base.forms import ThesaurusAvailableForm, THESAURUS_RESULT_LIST_SEPERATOR +from geonode.resource.manager import resource_manager + +test_image = Image.new("RGBA", size=(50, 50), color=(155, 0, 0)) + + +class ThumbnailTests(GeoNodeBaseTestSupport): + def setUp(self): + super().setUp() + self.rb = ResourceBase.objects.create(uuid=str(uuid4()), owner=get_user_model().objects.get(username="admin")) + + def tearDown(self): + super().tearDown() + + def test_initial_behavior(self): + """ + Tests that an empty resource has a missing image as null. + """ + self.assertFalse(self.rb.has_thumbnail()) + missing = self.rb.get_thumbnail_url() + self.assertIsNone(missing) + + def test_empty_image(self): + """ + Tests that an empty image does not change the current resource thumbnail. + """ + current = self.rb.get_thumbnail_url() + self.rb.save_thumbnail("test-thumb", None) + self.assertEqual(current, self.rb.get_thumbnail_url()) + + @patch("PIL.Image.open", return_value=test_image) + def test_monochromatic_image(self, image): + """ + Tests that an monochromatic image does not change the current resource thumbnail. + """ + filename = "test-thumb" + + current = self.rb.get_thumbnail_url() + self.rb.save_thumbnail(filename, image) + self.assertEqual(current, self.rb.get_thumbnail_url()) + + # cleanup: remove saved thumbnail + thumb_utils.remove_thumbs(filename) + self.assertFalse(thumb_utils.thumb_exists(filename)) + + @patch("PIL.Image.open", return_value=test_image) + def test_thumb_utils_methods(self, image): + """ + Bunch of tests on thumb_utils helpers. + """ + filename = "test-thumb" + upload_path = thumb_utils.thumb_path(filename) + self.assertEqual(upload_path, os.path.join(settings.THUMBNAIL_LOCATION, filename)) + thumb_utils.remove_thumbs(filename) + self.assertFalse(thumb_utils.thumb_exists(filename)) + f = BytesIO(test_image.tobytes()) + f.name = filename + storage_manager.save(upload_path, File(f)) + self.assertTrue(thumb_utils.thumb_exists(filename)) + self.assertEqual(thumb_utils.thumb_size(upload_path), 10000) + + # cleanup: remove saved thumbnail + thumb_utils.remove_thumbs(filename) + self.assertFalse(thumb_utils.thumb_exists(filename)) + + +class TestThumbnailUrl(GeoNodeBaseTestSupport): + def setUp(self): + super().setUp() + f = BytesIO(test_image.tobytes()) + f.name = "test_image.jpeg" + + +class TestCreationOfMissingMetadataAuthorsOrPOC(ThumbnailTests): + def test_add_missing_metadata_author_or_poc(self): + """ + Test that calling add_missing_metadata_author_or_poc resource method sets + a missing metadata_author and/or point of contact (poc) to resource owner + """ + user, _ = get_user_model().objects.get_or_create(username="zlatan_i") + + self.rb.owner = user + self.rb.add_missing_metadata_author_or_poc() + self.assertTrue("zlatan_i" in [author.username for author in self.rb.metadata_author]) + self.assertTrue("zlatan_i" in [author.username for author in self.rb.poc]) + + +class TestCreationOfContactRolesByDifferentInputTypes(ThumbnailTests): + """ + Test that contact roles can be set as people profile + """ + + def test_set_contact_role_as_people_profile(self): + user, _ = get_user_model().objects.get_or_create(username="zlatan_i") + + self.rb.owner = user + self.rb.metadata_author = user + self.rb.poc = user + self.rb.publisher = user + self.rb.custodian = user + self.rb.distributor = user + self.rb.resource_user = user + self.rb.resource_provider = user + self.rb.originator = user + self.rb.principal_investigator = user + self.rb.processor = user + + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.metadata_author]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.poc]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.publisher]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.custodian]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.distributor]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.resource_user]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.resource_provider]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.originator]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.principal_investigator]) + self.assertTrue("zlatan_i" in [cr.username for cr in self.rb.processor]) + + """ + Test that contact roles can be set as list of people profiles + """ + + def test_set_contact_role_as_list_of_people(self): + user, _ = get_user_model().objects.get_or_create(username="zlatan_i") + user2, _ = get_user_model().objects.get_or_create(username="sven_z") + + profile_list = [user, user2] + + self.rb.owner = user + self.rb.metadata_author = profile_list + self.rb.poc = profile_list + self.rb.publisher = profile_list + self.rb.custodian = profile_list + self.rb.distributor = profile_list + self.rb.resource_user = profile_list + self.rb.resource_provider = profile_list + self.rb.originator = profile_list + self.rb.principal_investigator = profile_list + self.rb.processor = profile_list + + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.metadata_author]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.poc]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.publisher]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.custodian]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.distributor]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.resource_user]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.resource_provider]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.originator]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.principal_investigator]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.processor]) + + """ + Test that contact roles can be set as queryset + """ + + def test_set_contact_role_as_queryset(self): + user, _ = get_user_model().objects.get_or_create(username="zlatan_i") + user2, _ = get_user_model().objects.get_or_create(username="sven_z") + + query = get_user_model().objects.filter(username__in=["zlatan_i", "sven_z"]) + + self.rb.owner = user + self.rb.metadata_author = query + self.rb.poc = query + self.rb.publisher = query + self.rb.custodian = query + self.rb.distributor = query + self.rb.resource_user = query + self.rb.resource_provider = query + self.rb.originator = query + self.rb.principal_investigator = query + self.rb.processor = query + + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.metadata_author]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.poc]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.publisher]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.custodian]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.distributor]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.resource_user]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.resource_provider]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.originator]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.principal_investigator]) + self.assertTrue("zlatan_i" and "sven_z" in [cr.username for cr in self.rb.processor]) + + +class RenderMenuTagTest(GeoNodeBaseTestSupport): + """ + Test class for render_menu and render_top_menu custom tags of base_tags + """ + + def setUp(self): + super().setUp() + self.placeholder_0 = MenuPlaceholder.objects.create(name="test_menu_placeholder_0") + self.placeholder_1 = MenuPlaceholder.objects.create(name="test_unicode_äöü_menu_placeholder_1") + self.menu_0_0 = Menu.objects.create(title="test_menu_0_0", order=0, placeholder=self.placeholder_0) + self.menu_0_1 = Menu.objects.create(title="test_menu_0_1", order=1, placeholder=self.placeholder_0) + self.menu_1_0 = Menu.objects.create(title="test_unicode_äöü_menu_1_0", order=0, placeholder=self.placeholder_1) + self.menu_item_0_0_0 = MenuItem.objects.create( + title="test_menu_item_0_0_0", order=0, blank_target=False, url="/about", menu=self.menu_0_0 + ) + self.menu_item_0_0_1 = MenuItem.objects.create( + title="test_menu_item_0_0_1", order=1, blank_target=False, url="/about", menu=self.menu_0_0 + ) + self.menu_item_0_1_0 = MenuItem.objects.create( + title="test_menu_item_0_1_0", order=0, blank_target=False, url="/about", menu=self.menu_0_1 + ) + self.menu_item_0_1_1 = MenuItem.objects.create( + title="test_menu_item_0_1_1", order=1, blank_target=False, url="/about", menu=self.menu_0_1 + ) + self.menu_item_0_1_2 = MenuItem.objects.create( + title="test_menu_item_0_1_2", order=2, blank_target=False, url="/about", menu=self.menu_0_1 + ) + self.menu_item_1_0_0 = MenuItem.objects.create( + title="test_unicode_äöü_menu_item_1_0_0", order=0, blank_target=False, url="/about", menu=self.menu_1_0 + ) + self.menu_item_1_0_1 = MenuItem.objects.create( + title="test_unicode_äöü_menu_item_1_0_1", order=1, blank_target=False, url="/about", menu=self.menu_1_0 + ) + + def test_get_menu_placeholder_0(self): + template = Template("{% load base_tags %} {% get_menu 'test_menu_placeholder_0' %}") + rendered = template.render(Context({})) + # menu_placeholder_0 + # first menu with ascii chars + self.assertIn( + self.menu_0_0.title, rendered, f'Expected "{self.menu_0_0.title}" string in the rendered template' + ) + self.assertIn( + self.menu_item_0_0_0.title, + rendered, + f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_0_0_1.title, + rendered, + f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template', + ) + # second menu + self.assertIn( + self.menu_0_1.title, rendered, f'Expected "{self.menu_0_1.title}" string in the rendered template' + ) + self.assertIn( + self.menu_item_0_1_0.title, + rendered, + f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_0_1_1.title, + rendered, + f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_0_1_2.title, + rendered, + f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template', + ) + # menu_placeholder_1 + # first menu + # unicode + self.assertNotIn( + self.menu_1_0.title, rendered, f'No "{self.menu_1_0.title}" string expected in the rendered template' + ) + self.assertNotIn( + self.menu_item_1_0_0.title, + rendered, + f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_1_0_1.title, + rendered, + f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template', + ) + + def test_get_menu_placeholder_1(self): + template = Template("{% load base_tags %} {% get_menu 'test_unicode_äöü_menu_placeholder_1' %}") + rendered = template.render(Context({})) + # menu_placeholder_0 + # first menu + self.assertNotIn( + self.menu_0_0.title, rendered, f'No "{self.menu_0_0.title}" string expected in the rendered template' + ) + self.assertNotIn( + self.menu_item_0_0_0.title, + rendered, + f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_0_0_1.title, + rendered, + f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template', + ) + # second menu + self.assertNotIn( + self.menu_0_1.title, rendered, f'No "{self.menu_0_1.title}" string expected in the rendered template' + ) + self.assertNotIn( + self.menu_item_0_1_0.title, + rendered, + f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_0_1_1.title, + rendered, + f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_0_1_2.title, + rendered, + f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template', + ) + # menu_placeholder_1 + # first menu + # unicode + self.assertIn( + self.menu_1_0.title, rendered, f'Expected "{self.menu_1_0.title}" string in the rendered template' + ) + self.assertIn( + self.menu_item_1_0_0.title, + rendered, + f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_1_0_1.title, + rendered, + f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template', + ) + + def test_render_nav_menu_placeholder_0(self): + template = Template("{% load base_tags %} {% render_nav_menu 'test_menu_placeholder_0' %}") + rendered = template.render(Context({})) + # menu_placeholder_0 + # first menu + self.assertIn( + self.menu_0_0.title, rendered, f'Expected "{self.menu_0_0.title}" string in the rendered template' + ) + self.assertIn( + self.menu_item_0_0_0.title, + rendered, + f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_0_0_1.title, + rendered, + f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template', + ) + # second menu + self.assertIn( + self.menu_0_1.title, rendered, f'Expected "{self.menu_0_1.title}" string in the rendered template' + ) + self.assertIn( + self.menu_item_0_1_0.title, + rendered, + f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_0_1_1.title, + rendered, + f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_0_1_2.title, + rendered, + f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template', + ) + # menu_placeholder_1 + # first menu + # unicode + self.assertNotIn( + self.menu_1_0.title, rendered, f'No "{self.menu_1_0.title}" string expected in the rendered template' + ) + self.assertNotIn( + self.menu_item_1_0_0.title, + rendered, + f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_1_0_1.title, + rendered, + f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template', + ) + + def test_render_nav_menu_placeholder_1(self): + template = Template("{% load base_tags %} {% render_nav_menu 'test_unicode_äöü_menu_placeholder_1' %}") + rendered = template.render(Context({})) + # menu_placeholder_0 + # first menu + self.assertNotIn( + self.menu_0_0.title, rendered, f'No "{self.menu_0_0.title}" string expected in the rendered template' + ) + self.assertNotIn( + self.menu_item_0_0_0.title, + rendered, + f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_0_0_1.title, + rendered, + f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template', + ) + # second menu + self.assertNotIn( + self.menu_0_1.title, rendered, f'No "{self.menu_0_1.title}" string expected in the rendered template' + ) + self.assertNotIn( + self.menu_item_0_1_0.title, + rendered, + f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_0_1_1.title, + rendered, + f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template', + ) + self.assertNotIn( + self.menu_item_0_1_2.title, + rendered, + f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template', + ) + # menu_placeholder_1 + # first menu + # unicode + self.assertIn( + self.menu_1_0.title, rendered, f'Expected "{self.menu_1_0.title}" string in the rendered template' + ) + self.assertIn( + self.menu_item_1_0_0.title, + rendered, + f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template', + ) + self.assertIn( + self.menu_item_1_0_1.title, + rendered, + f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template', + ) + + +class DeleteResourcesCommandTests(GeoNodeBaseTestSupport): + def test_delete_resources_no_arguments(self): + args = [] + kwargs = {} + + with self.assertRaises(CommandError) as exception: + call_command("delete_resources", *args, **kwargs) + + self.assertIn( + "No configuration provided", exception.exception.args[0], '"No configuration" exception expected.' + ) + + def test_delete_resources_too_many_arguments(self): + args = [] + kwargs = {"config_path": "/example/config.txt", "map_filters": "*"} + + with self.assertRaises(CommandError) as exception: + call_command("delete_resources", *args, **kwargs) + + self.assertIn( + "Too many configuration options provided", + exception.exception.args[0], + '"Too many configuration options provided" exception expected.', + ) + + def test_delete_resource_config_file_not_existing(self): + args = [] + kwargs = {"config_path": "/example/config.json"} + + with self.assertRaises(CommandError) as exception: + call_command("delete_resources", *args, **kwargs) + + self.assertIn( + "Specified configuration file does not exist", + exception.exception.args[0], + '"Specified configuration file does not exist" exception expected.', + ) + + def test_delete_resource_config_file_empty(self): + # create an empty config file + config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "delete_resources_config.json") + open(config_file_path, "a").close() + + args = [] + kwargs = {"config_path": config_file_path} + + with self.assertRaises(CommandError) as exception: + call_command("delete_resources", *args, **kwargs) + + self.assertIn( + "Specified configuration file is empty", + exception.exception.args[0], + '"Specified configuration file is empty" exception expected.', + ) + + # delete the config file + os.remove(config_file_path) + + +class ConfigurationTest(GeoNodeBaseTestSupport): + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_read_only_whitelist(self): + web_client = Client() + + # set read-only flag + config = Configuration.load() + config.read_only = True + config.maintenance = False + config.save() + + # post to whitelisted URLs as AnonymousUser + for url_name in ReadOnlyMiddleware.WHITELISTED_URL_NAMES: + if url_name == "login": + response = web_client.post(reverse("admin:login")) + elif url_name == "logout": + response = web_client.post(reverse("admin:logout")) + else: + response = web_client.post(reverse(url_name)) + + self.assertNotEqual(response.status_code, 405, "Whitelisted URL is not available.") + + def test_read_only_casual_user_privileges(self): + web_client = Client() + url_name = "autocomplete_region" + + # set read-only flag + config = Configuration.load() + config.read_only = True + config.maintenance = False + config.save() + + # get user + user, _ = get_user_model().objects.get_or_create(username="user1") + web_client.force_login(user) + + # post not whitelisted URL as superuser + response = web_client.post(reverse(url_name)) + + self.assertEqual(response.status_code, 405, "User is allowed to post to forbidden URL") + + def test_maintenance_whitelist(self): + web_client = Client() + + # set read-only flag + config = Configuration.load() + config.read_only = False + config.maintenance = True + config.save() + + # post to whitelisted URLs as AnonymousUser + for url_name in MaintenanceMiddleware.WHITELISTED_URL_NAMES: + if url_name == "login": + response = web_client.get(reverse("admin:login")) + elif url_name == "logout": + response = web_client.get(reverse("admin:logout")) + elif url_name == "index": + # url needed in the middleware only for admin panel login redirection + continue + else: + response = web_client.get(reverse(url_name)) + + self.assertNotEqual(response.status_code, 503, "Whitelisted URL is not available.") + + def test_maintenance_false(self): + web_client = Client() + + # set read-only flag + config = Configuration.load() + config.read_only = False + config.maintenance = False + config.save() + + # post not whitelisted URL as superuser + response = web_client.get("/") + + self.assertNotEqual(response.status_code, 503, "User is allowed to get index page") + + def test_maintenance_true(self): + web_client = Client() + + # set read-only flag + config = Configuration.load() + config.read_only = False + config.maintenance = True + config.save() + + # post not whitelisted URL as superuser + response = web_client.get("/") + + self.assertEqual(response.status_code, 503, "User is allowed to get index page") + + @patch.dict(os.environ, {"FORCE_READ_ONLY_MODE": "True"}) + def test_readonly_overwrite_by_env(self): + config = Configuration.load() + self.assertTrue(config.read_only) + + @patch.dict(os.environ, {"FORCE_READ_ONLY_MODE": "False"}) + def test_readonly_is_not_overwrite_by_env(self): + # will take the value from the db + config = Configuration.load() + self.assertFalse(config.read_only) + + +class TestOwnerRightsRequestUtils(TestCase): + def setUp(self): + User = get_user_model() + self.user = User.objects.create(username="test", email="test@test.com") + self.admin = User.objects.create(username="admin", email="test@test.com", is_superuser=True) + self.d = Document.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) + self.la = Dataset.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) + self.s = Service.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) + self.m = Map.objects.create(uuid=str(uuid4()), owner=self.user, title="test", is_approved=True) + + def test_get_concrete_resource(self): + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.d.id)), Document) + ) + + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.la.id)), Dataset) + ) + + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.s.id)), Service) + ) + + self.assertTrue( + isinstance(OwnerRightsRequestViewUtils.get_resource(ResourceBase.objects.get(pk=self.m.id)), Map) + ) + + @override_settings(ADMIN_MODERATE_UPLOADS=True) + def test_msg_recipients_admin_mode(self): + users_count = 1 + self.assertEqual(users_count, OwnerRightsRequestViewUtils.get_message_recipients(self.user).count()) + + @override_settings(ADMIN_MODERATE_UPLOADS=False) + def test_msg_recipients_workflow_off(self): + users_count = 0 + self.assertEqual(users_count, OwnerRightsRequestViewUtils.get_message_recipients(self.user).count()) + + @override_settings(ADMIN_MODERATE_UPLOADS=True) + def test_display_change_perms_button_tag_moderated(self): + admin_perms = display_change_perms_button(self.la, self.admin, {}) + user_perms = display_change_perms_button(self.la, self.user, {}) + self.assertTrue(admin_perms) + self.assertFalse(user_perms) + + @override_settings(ADMIN_MODERATE_UPLOADS=False) + def test_display_change_perms_button_tag_standard(self): + admin_perms = display_change_perms_button(self.la, self.admin, {}) + user_perms = display_change_perms_button(self.la, self.user, {}) + self.assertTrue(admin_perms) + self.assertTrue(user_perms) + + +class SanitizedHtmlTest(TestCase): + def test_sanitized_html(self): + test_html = "Allowed Text" + expected_output = "Allowed Text" + sanitized_output = sanitize_html(test_html) + self.assertEqual(sanitized_output, expected_output) + + +class TestGetVisibleResource(TestCase): + def setUp(self): + self.user = get_user_model().objects.create(username="mikel_arteta") + self.category = TopicCategory.objects.create(identifier="biota") + self.rb = ResourceBase.objects.create(uuid=str(uuid4()), category=self.category, owner=self.user) + + def test_category_data_not_shown_for_missing_resourcebase_permissions(self): + """ + Test that a user without view permissions of a resource base does not see + ISO category format data of the ISO category + """ + categories = get_visibile_resources(self.user) + self.assertEqual(categories["iso_formats"].count(), 0) + + def test_category_data_shown_for_with_resourcebase_permissions(self): + """ + Test that a user with view permissions of a resource base can see + ISO format data of the ISO category + """ + assign_perm("view_resourcebase", self.user, self.rb) + categories = get_visibile_resources(self.user) + self.assertEqual(categories["iso_formats"].count(), 1) + + def test_visible_notifications(self): + """ + Test that a standard user won't be able to show ADMINS_ONLY_NOTICE_TYPES + """ + self.assertFalse(show_notification("monitoring_alert", self.user)) + self.assertTrue(show_notification("request_download_resourcebase", self.user)) + + def test_extent_filter_crossing_dateline(self): + from .bbox_utils import filter_bbox + + _ll = None + try: + bbox = [166.06619, -22.40043, 172.09202, -13.03425] + _ll = Dataset.objects.create( + uuid=str(uuid4()), + owner=self.user, + name="test_extent_filter_crossing_dateline", + title="test_extent_filter_crossing_dateline", + alternate="geonode:test_extent_filter_crossing_dateline", + is_approved=True, + is_published=True, + ll_bbox_polygon=Polygon.from_bbox(bbox), + ) + self.assertListEqual(list(_ll.ll_bbox_polygon.extent), bbox, _ll.ll_bbox_polygon.extent) + self.assertTrue(Dataset.objects.filter(title=_ll.title).exists(), Dataset.objects.all()) + _qs = filter_bbox( + Dataset.objects.all(), "-180.0000,-39.7790,-164.2456,9.2702,134.0552,-39.7790,180.0000,9.2702" + ) + self.assertTrue(_qs.filter(title=_ll.title), Dataset.objects.all() | _qs.all()) + finally: + if _ll: + _ll.delete() + + +class TestHtmlTagRemoval(SimpleTestCase): + def test_not_tags_in_attribute(self): + attribute_target_value = "This is not a templated text" + r = ResourceBase() + filtered_value = r._remove_html_tags(attribute_target_value) + self.assertEqual(attribute_target_value, filtered_value) + + def test_simple_tags_in_attribute(self): + tagged_value = "

    This is not a templated text

    " + attribute_target_value = "This is not a templated text" + r = ResourceBase() + filtered_value = r._remove_html_tags(tagged_value) + self.assertEqual(filtered_value, attribute_target_value) + + def test_complex_tags_in_attribute(self): + tagged_value = """

    +

    Something in ícontainer

    £682m

    """ + attribute_target_value = """This is not a templated text Something in ícontainer £682m""" + r = ResourceBase() + filtered_value = r._remove_html_tags(tagged_value) + self.assertEqual(filtered_value, attribute_target_value) + + def test_converted_html_in_tags_with_char_references(self): + tagged_value = """

    <p>Abstract value & some text</p>

    """ + attribute_target_value = """Abstract value & some text""" + r = ResourceBase() + filtered_value = r._remove_html_tags(tagged_value) + self.assertEqual(filtered_value, attribute_target_value) + + def test_converted_html_in_tags_with_with_multiple_tags(self): + tagged_value = """

    Abstract value & some text

    """ + attribute_target_value = """Abstract value & some text""" + r = ResourceBase() + filtered_value = r._remove_html_tags(tagged_value) + self.assertEqual(filtered_value, attribute_target_value) + + +class TestTagThesaurus(TestCase): + # loading test thesausurs + fixtures = ["test_thesaurus.json"] + + def setUp(self): + self.sut = Thesaurus( + identifier="foo_name", + title="GEMET - INSPIRE themes, version 1.0", + date="2018-05-23T10:25:56", + description="GEMET - INSPIRE themes, version 1.0", + slug="", + about="http://inspire.ec.europa.eu/theme", + ) + self.tkeywords = ThesaurusKeyword.objects.all() + + def test_get_unique_thesaurus_list(self): + actual = get_unique_thesaurus_set(self.tkeywords) + self.assertSetEqual({1, 3}, actual) + + def test_get_thesaurus_title(self): + tid = self.__get_last_thesaurus().id + actual = get_thesaurus_title(tid) + self.assertEqual(self.sut.title, actual) + + def test_get_thesaurus_date(self): + tid = self.__get_last_thesaurus().id + actual = get_thesaurus_date(tid) + self.assertEqual(self.sut.date, actual) + + def test_get_name_translation_raise_exception_if_identifier_does_not_exists(self): + with self.assertRaises(ObjectDoesNotExist): + get_name_translation("foo_indentifier") + + @patch("geonode.base.templatetags.thesaurus.get_language") + def test_get_name_translation_return_thesauro_title_if_label_for_selected_language_does_not_exists(self, lang): + lang.return_value = "ke" + actual = get_name_translation("inspire-theme") + expected = "GEMET - INSPIRE themes, version 1.0" + self.assertEqual(expected, actual) + + @patch("geonode.base.templatetags.thesaurus.get_language") + def test_get_thesaurus_translation_by_id(self, lang): + lang.return_value = "it" + actual = get_thesaurus_translation_by_id(1) + expected = "Tema GEMET - INSPIRE, versione 1.0" + self.assertEqual(expected, actual) + + @patch("geonode.base.templatetags.thesaurus.get_language") + def test_get_thesaurus_localized_label(self, lang): + lang.return_value = "de" + keyword = ThesaurusKeyword.objects.get(id=1) + actual = get_thesaurus_localized_label(keyword) + expected = "Adressen" + self.assertEqual(expected, actual) + + @patch("geonode.base.templatetags.thesaurus.get_language") + def test_get_name_translation_return_label_title_if_label_for_selected_language_exists(self, lang): + lang.return_value = "it" + actual = get_name_translation("inspire-theme") + expected = "Tema GEMET - INSPIRE, versione 1.0" + self.assertEqual(expected, actual) + + @staticmethod + def __get_last_thesaurus(): + return Thesaurus.objects.all().order_by("id")[0] + + +@override_settings(THESAURUS_DEFAULT_LANG="en") +class TestThesaurusAvailableForm(TestCase): + # loading test thesausurs + fixtures = ["test_thesaurus.json"] + + def setUp(self): + self.sut = ThesaurusAvailableForm + + def test_form_is_valid_if_all_fields_are_missing(self): + # is now always true since the required is moved to the UI + # (like the other fields) + actual = self.sut(data={}) + self.assertTrue(actual.is_valid()) + + def test_form_is_invalid_if_fileds_send_unexpected_values(self): + actual = self.sut(data={"1": [1, 2]}) + self.assertFalse(actual.is_valid()) + + def test_form_is_valid_if_fileds_send_expected_values(self): + actual = self.sut(data={"1": 1}) + self.assertTrue(actual.is_valid()) + + def test_field_class_treq_is_correctly_set_when_field_is_required(self): + actual = self.sut(data={"1": 1}) + required = actual.fields.get("1") + obj_class = required.widget.attrs.get("class") + self.assertTrue(obj_class == "treq") + + def test_field_class_treq_is_not_set_when_field_is_optional(self): + actual = self.sut(data={"1": 1}) + required = actual.fields.get("2") + obj_class = required.widget.attrs.get("class") + self.assertTrue(obj_class == "") + + def test_will_return_thesaurus_with_the_expected_defined_order(self): + actual = self.sut(data={"1": 1}) + fields = list(actual.fields.items()) + # will check if the first element of the tuple is the thesaurus_id = 2 + self.assertEqual(fields[0][0], "2") + # will check if the second element of the tuple is the thesaurus_id = 1 + self.assertEqual(fields[1][0], "1") + + def test_will_return_thesaurus_with_the_defaul_order_as_0(self): + # Update thesaurus order to 0 in order to check if the default order by id is observed + t = Thesaurus.objects.get(identifier="inspire-theme") + t.order = 0 + t.save() + actual = ThesaurusAvailableForm(data={"1": 1}) + fields = list(actual.fields.items()) + # will check if the first element of the tuple is the thesaurus_id = 2 + self.assertEqual(fields[0][0], "1") + # will check if the second element of the tuple is the thesaurus_id = 1 + self.assertEqual(fields[1][0], "2") + + def test_get_thesuro_key_label_with_cmd_language_code(self): + # in python test language code look like 'en' this test checks if key label result function + # returns correct results + tid = 1 + translation.activate("en") + t_available_form = ThesaurusAvailableForm(data={"1": tid}) + results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language()) + self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR) + + def test_get_thesuro_key_label_with_browser_language_code(self): + # in browser scenario language does not look like "it", but rather include coutry code + # like "it-it" this test checks if _get_thesauro_keyword_label can handle this + tid = 1 + translation.activate("en-us") + t_available_form = ThesaurusAvailableForm(data={"1": tid}) + results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language()) + self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR) + + +class TestFacets(TestCase): + def setUp(self): + self.user = get_user_model().objects.create(username="test", email="test@test.com") + Dataset.objects.update_or_create( + name="test_boxes_vector", + defaults=dict( + uuid=str(uuid4()), + owner=self.user, + title="test_boxes", + abstract="nothing", + subtype="vector", + is_approved=True, + ), + ) + Dataset.objects.update_or_create( + name="test_1_vector", + defaults=dict( + uuid=str(uuid4()), + owner=self.user, + title="test_1", + abstract="contains boxes", + subtype="vector", + is_approved=True, + ), + ) + Dataset.objects.update_or_create( + name="test_2_vector", + defaults=dict( + uuid=str(uuid4()), + owner=self.user, + title="test_2", + purpose="contains boxes", + subtype="vector", + is_approved=True, + ), + ) + Dataset.objects.update_or_create( + name="test_3_vector", + defaults=dict(uuid=str(uuid4()), owner=self.user, title="test_3", subtype="vector", is_approved=True), + ) + Dataset.objects.update_or_create( + name="test_boxes_vector", + defaults=dict( + uuid=str(uuid4()), + owner=self.user, + title="test_boxes", + abstract="nothing", + subtype="vector", + is_approved=True, + ), + ) + Dataset.objects.update_or_create( + name="test_1_raster", + defaults=dict( + uuid=str(uuid4()), + owner=self.user, + title="test_1", + abstract="contains boxes", + subtype="raster", + is_approved=True, + ), + ) + Dataset.objects.update_or_create( + name="test_2_raster", + defaults=dict( + uuid=str(uuid4()), + owner=self.user, + title="test_2", + purpose="contains boxes", + subtype="raster", + is_approved=True, + ), + ) + Dataset.objects.update_or_create( + name="test_boxes_raster", + defaults=dict(uuid=str(uuid4()), owner=self.user, title="test_boxes", subtype="raster", is_approved=True), + ) + + self.request_mock = Mock(spec=requests.Request, GET=Mock()) + + def test_facets_filter_datasets_returns_correctly(self): + for _l in Dataset.objects.all(): + _l.set_default_permissions() + _l.clear_dirty_state() + _l.set_processing_state(enumerations.STATE_PROCESSED) + self.request_mock.GET.get.side_effect = lambda key, self: { + "title__icontains": "boxes", + "abstract__icontains": "boxes", + "purpose__icontains": "boxes", + "date__gte": None, + "date__range": None, + "date__lte": None, + "extent": None, + }.get(key) + self.request_mock.GET.getlist.return_value = None + self.request_mock.user = self.user + results = facets({"request": self.request_mock}) + self.assertEqual(results["vector"], 3) + self.assertEqual(results["raster"], 3) + + +class TestGenerateThesaurusReference(TestCase): + fixtures = ["test_thesaurus.json"] + + def setUp(self): + self.site_url = settings.SITEURL if hasattr(settings, "SITEURL") else "http://localhost" + + """ + If the keyword.about does not exists, the url created will have a prefix and a specifier: + as prefix: + - use the Keyword's thesaurus.about URI if it exists, + - otherwise use as prefix the geonode site URL composed with some thesaurus info: f'{settings.SITEURL}/thesaurus/{thesaurus.identifier}' + as specifier: + - we may use the ThesaurusKeyword.alt_label if it exists, otherwise its id + + So the final about field value will be composed as f'{prefix}#{specifier}' + """ + + def test_should_return_keyword_url(self): + expected = "http://inspire.ec.europa.eu/theme/ad" + keyword = ThesaurusKeyword.objects.get(id=1) + actual = generate_thesaurus_reference(keyword) + keyword.refresh_from_db() + """ + Check if the expected about has been created and that the instance is correctly updated + """ + self.assertEqual(expected, actual) + self.assertEqual(expected, keyword.about) + + def test_should_return_as_url_thesaurus_about_and_keyword_alt_label(self): + expected = "http://inspire.ec.europa.eu/theme#foo_keyword" + keyword = ThesaurusKeyword.objects.get(alt_label="foo_keyword") + actual = generate_thesaurus_reference(keyword) + keyword.refresh_from_db() + """ + Check if the expected about has been created and that the instance is correctly updated + """ + self.assertEqual(expected, actual) + self.assertEqual(expected, keyword.about) + + def test_should_return_as_url_thesaurus_about_and_keyword_id(self): + expected = "http://inspire.ec.europa.eu/theme#37" + keyword = ThesaurusKeyword.objects.get(id=37) + actual = generate_thesaurus_reference(keyword) + keyword.refresh_from_db() + """ + Check if the expected about has been created and that the instance is correctly updated + """ + self.assertEqual(expected, actual) + self.assertEqual(expected, keyword.about) + + def test_should_return_as_url_site_url_and_keyword_label(self): + expected = f"{self.site_url}/thesaurus/no-about-thesauro#bar_keyword" + keyword = ThesaurusKeyword.objects.get(id=39) + actual = generate_thesaurus_reference(keyword) + keyword.refresh_from_db() + """ + Check if the expected about has been created and that the instance is correctly updated + """ + self.assertEqual(expected, actual) + self.assertEqual(expected, keyword.about) + + def test_should_return_as_url_site_url_and_keyword_id(self): + expected = f"{self.site_url}/thesaurus/no-about-thesauro#38" + keyword = ThesaurusKeyword.objects.get(id=38) + actual = generate_thesaurus_reference(keyword) + keyword.refresh_from_db() + """ + Check if the expected about has been created and that the instance is correctly updated + """ + self.assertEqual(expected, actual) + self.assertEqual(expected, keyword.about) + + +class TestHandleMetadataKeyword(TestCase): + fixtures = ["test_thesaurus.json"] + + def setUp(self): + self.keyword = [ + { + "keywords": ["features", "test_dataset"], + "thesaurus": {"date": None, "datetype": None, "title": None}, + "type": "theme", + }, + { + "keywords": ["no conditions to access and use"], + "thesaurus": { + "date": "2020-10-30T16:58:34", + "datetype": "publication", + "title": "Test for ordering", + }, + "type": None, + }, + { + "keywords": ["ad", "af"], + "thesaurus": { + "date": "2008-06-01", + "datetype": "publication", + "title": "GEMET - INSPIRE themes, version 1.0", + }, + "type": None, + }, + {"keywords": ["Global"], "thesaurus": {"date": None, "datetype": None, "title": None}, "type": "place"}, + ] + self.dataset = create_single_dataset("keyword-handler") + self.sut = KeywordHandler(instance=self.dataset, keywords=self.keyword) + + def test_return_empty_if_keywords_is_an_empty_list(self): + setattr(self.sut, "keywords", []) + keyword, thesaurus_keyword = self.sut.handle_metadata_keywords() + self.assertListEqual([], keyword) + self.assertListEqual([], thesaurus_keyword) + + def test_should_return_the_expected_keyword_extracted_from_raw_and_the_thesaurus_keyword(self): + keyword, thesaurus_keyword = self.sut.handle_metadata_keywords() + self.assertSetEqual({"features", "test_dataset", "no conditions to access and use"}, set(keyword)) + self.assertListEqual(["ad", "af"], [x.alt_label for x in thesaurus_keyword]) + + def test_should_assign_correclty_the_keyword_to_the_dataset_object(self): + self.sut.set_keywords() + current_keywords = [keyword.name for keyword in self.dataset.keywords.all()] + current_tkeyword = [t.alt_label for t in self.dataset.tkeywords.all()] + self.assertSetEqual({"features", "test_dataset", "no conditions to access and use"}, set(current_keywords)) + self.assertSetEqual({"ad", "af"}, set(current_tkeyword)) + + def test_is_thesaurus_available_should_return_queryset_with_existing_thesaurus(self): + keyword = "ad" + thesaurus = {"title": "GEMET - INSPIRE themes, version 1.0"} + actual = self.sut.is_thesaurus_available(thesaurus, keyword) + self.assertEqual(1, len(actual)) + + def test_is_thesaurus_available_should_return_empty_queryset_for_non_existing_thesaurus(self): + keyword = "ad" + thesaurus = {"title": "Random Thesaurus Title"} + actual = self.sut.is_thesaurus_available(thesaurus, keyword) + self.assertEqual(0, len(actual)) + + +class Test_HierarchicalTagManager(GeoNodeBaseTestSupport): + def setUp(self) -> None: + self.sut = create_single_dataset(name="dataset_for_keyword") + self.keyword = HierarchicalKeyword.objects.create(name="test_kw", slug="test_kw", depth=1) + self.keyword.save() + + def tearDown(self) -> None: + self.sut.keywords.remove(self.keyword) + + def test_keyword_are_correctly_saved(self): + self.assertFalse(self.sut.keywords.exists()) + self.sut.keywords.add(self.keyword) + self.assertTrue(self.sut.keywords.exists()) + + @patch("django.db.models.query.QuerySet._fetch_all") + def test_keyword_raise_integrity_error(self, keyword_fetch_method): + keyword_fetch_method.side_effect = IntegrityError() + logger = logging.getLogger("geonode.base.models") + with self.assertLogs(logger, level="WARNING") as _log: + self.sut.keywords.add(self.keyword) + self.assertIn("The keyword provided already exists", [x.message for x in _log.records]) + + @patch("geonode.base.models.HierarchicalKeyword.add_root") + def test_keyword_raise_db_error(self, add_root_mocked): + add_root_mocked.side_effect = OperationalError() + logger = logging.getLogger("geonode.base.models") + with self.assertLogs(logger) as _log: + self.sut.keywords.add("keyword2") + self.assertIn( + "Error during the keyword creation for keyword: keyword2", + [x.message for x in _log.records], + ) + + +def _cache_less_call_storers(instance, custom={}): + storer_module_path = settings.METADATA_STORERS if hasattr(settings, "METADATA_STORERS") else [] + storers = [import_string(storer_path) for storer_path in storer_module_path] + for storer in storers: + storer(instance, custom) + return instance + + +class TestRegions(GeoNodeBaseTestSupport): + def setUp(self): + self.dataset_inside_region = GEOSGeometry( + "POLYGON ((-4.01799226543944599 57.18451093931114571, 8.89409253052255622 56.91828238681708285, \ + 9.29343535926363984 47.73339732577194638, -3.75176371294537603 48.13274015451304422, \ + -4.01799226543944599 57.18451093931114571))", + srid=4326, + ) + + self.dataset_overlapping_region = GEOSGeometry( + "POLYGON ((15.28357779038003628 33.6232840435866791, 28.19566258634203848 33.35705549109261625, \ + 28.5950054150831221 24.17217043004747978, 15.54980634287410624 24.57151325878857762, \ + 15.28357779038003628 33.6232840435866791))", + srid=4326, + ) + + self.dataset_outside_region = GEOSGeometry( + "POLYGON ((-3.75176371294537603 23.10725622007123548, 9.16032108301662618 22.84102766757717262, \ + 9.5596639117577098 13.65614260653203615, -3.48553516045130607 14.05548543527313399, \ + -3.75176371294537603 23.10725622007123548))", + srid=4326, + ) + + def test_region_assignment_for_extent(self): + region = Region.objects.get(code="EUR") + + self.assertTrue( + region.is_assignable_to_geom(self.dataset_inside_region), "Extent inside a region shouldn't be assigned" + ) + self.assertTrue( + region.is_assignable_to_geom(self.dataset_overlapping_region), + "Extent overlapping a region should be assigned", + ) + self.assertFalse( + region.is_assignable_to_geom(self.dataset_outside_region), "Extent outside a region should be assigned" + ) + + @patch("geonode.resource.utils.call_storers", _cache_less_call_storers) + def test_regions_are_assigned_if_handler_is_used(self): + with override_settings(METADATA_STORERS=["geonode.resource.regions_storer.spatial_predicate_region_assignor"]): + dataset = resource_manager.create( + None, + resource_type=Dataset, + defaults=dict(owner=get_user_model().objects.first(), title="test_region_dataset", is_approved=True), + ) + self.assertTrue(dataset.regions.exists()) + self.assertEqual(1, dataset.regions.count()) + self.assertEqual("Global", dataset.regions.first().name) + + +class TestMetadataStorer(GeoNodeBaseTestSupport): + + @patch("geonode.resource.utils.call_storers", _cache_less_call_storers) + def test_create_passing_custom_to_post_save(self): + + with override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]): + User = get_user_model() + user = User.objects.create(username="test", email="test@test.com") + license = License.objects.all().first() + group = Group.objects.all().first() + dataset = resource_manager.create( + str(uuid4()), + resource_type=Dataset, + defaults=dict(owner=user, title="test"), + custom=dict(group=group.pk, license=license), + ) + self.assertIsNotNone(dataset.license) + self.assertIsNotNone(dataset.group) + self.assertEqual(group.pk, dataset.group.pk) + + +class LinkedResourcesTest(GeoNodeBaseTestSupport): + def test_autocomplete_linked_resource(self): + d = [] + try: + user, _ = get_user_model().objects.get_or_create(username="admin") + + for t in ("dataset1", "dataset2", "other"): + d.append(ResourceBase.objects.create(title=t, owner=user, is_approved=True, is_published=True)) + + web_client = Client() + web_client.force_login(user) + url_name = "autocomplete_linked_resource" + + # get all resources + response = web_client.get(reverse(url_name)) + rjson = response.json() + + self.assertEqual(response.status_code, 200, "Can not get autocomplete API") + self.assertIn("results", rjson, "Can not find results") + self.assertEqual(len(rjson["results"]), 3, "Unexpected results count") + + # filter by title + response = web_client.get( + reverse(url_name), + data={ + "q": "dataset", + }, + ) + rjson = response.json() + self.assertEqual(len(rjson["results"]), 2, "Unexpected results count") + + # filter by title, exclude + response = web_client.get( + reverse(url_name), data={"q": "dataset", "forward": json.dumps({"exclude": d[0].id})} + ) + rjson = response.json() + self.assertEqual(len(rjson["results"]), 1, "Unexpected results count") + + finally: + for _ in d: + _.delete() + + +class TestResourceBaseViewSetQueryset(TestCase): + @classmethod + def setUpTestData(cls): + cls.test_user = get_user_model().objects.create_user( + username="test_queryset_user", email="test@example.com", password="testpass123" + ) + + cls.admin_user = get_user_model().objects.create_user( + username="admin_queryset_user", email="admin@example.com", password="adminpass123", is_staff=True + ) + + cls.test_resources = [] + for i in range(5): + resource = ResourceBase.objects.create( + title=f"test_resource_queryset_{i}", + uuid=str(uuid4()), + owner=cls.test_user if i % 2 == 0 else cls.admin_user, + abstract=f"Test resource {i} for queryset", + subtype="vector", + is_approved=True, + is_published=True, + ) + cls.test_resources.append(resource) + + def test_original_queryset_vs_optimized_queryset(self): + """Test that original and optimized querysets return the same results""" + + original_queryset = ResourceBase.objects.all().order_by("-created") + + optimized_queryset = ResourceBase.objects.select_related("owner").order_by("-created") + + original_list = list(original_queryset) + optimized_list = list(optimized_queryset) + + self.assertEqual(len(original_list), len(optimized_list)) + + self.assertEqual(original_list, optimized_list) + + original_pks = [obj.pk for obj in original_list] + optimized_pks = [obj.pk for obj in optimized_list] + self.assertEqual(original_pks, optimized_pks) + + +class FixOtherRestrictionsTest(GeoNodeBaseTestSupport): + + def test_fix_otherrestrictions_codetype(self): + from geonode.base.models import RestrictionCodeType + from django.db.models import Q + + migration_module = importlib.import_module( + "geonode.base.migrations.0094_fix_otherrestrictions_codetype" + ) # importing migration module to test + fix_otherrestrictions_codetype = migration_module.fix_otherrestrictions_codetype + + updated = RestrictionCodeType.objects.filter( + Q(identifier="limitation not listed") | Q(identifier="otherRestrictions") + ).update( + identifier="limitation not listed", description="otherRestrictions", gn_description="otherRestrictions" + ) + + self.assertEqual(updated, 1, "Expected one record to be updated") + + fix_otherrestrictions_codetype(apps=apps, schema_editor=None) # calling the migration function to correct + + with self.assertRaises(RestrictionCodeType.DoesNotExist): + RestrictionCodeType.objects.get(identifier="limitation not listed") + + fixed_obj = RestrictionCodeType.objects.get(identifier="otherRestrictions") + self.assertEqual(fixed_obj.description, "limitation not listed") + self.assertEqual(fixed_obj.gn_description, "limitation not listed") diff --git a/geonode/base/tests/data/img.gif b/geonode/base/tests/data/img.gif new file mode 100644 index 0000000..56959b6 Binary files /dev/null and b/geonode/base/tests/data/img.gif differ diff --git a/geonode/base/translation.py b/geonode/base/translation.py new file mode 100644 index 0000000..ca171cc --- /dev/null +++ b/geonode/base/translation.py @@ -0,0 +1,69 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from modeltranslation.translator import translator, TranslationOptions +from geonode.base.models import ( + TopicCategory, + SpatialRepresentationType, + Region, + RestrictionCodeType, + License, + ResourceBase, +) + + +class TopicCategoryTranslationOptions(TranslationOptions): + fields = ( + "description", + "gn_description", + ) + + +class SpatialRepresentationTypeTranslationOptions(TranslationOptions): + fields = ( + "description", + "gn_description", + ) + + +class RegionTranslationOptions(TranslationOptions): + fields = ("name",) + + +class RestrictionCodeTypeTranslationOptions(TranslationOptions): + fields = ( + "description", + "gn_description", + ) + + +class LicenseTranslationOptions(TranslationOptions): + fields = ( + "name", + "description", + "license_text", + ) + + +translator.register(TopicCategory, TopicCategoryTranslationOptions) +translator.register(SpatialRepresentationType, SpatialRepresentationTypeTranslationOptions) +translator.register(Region, RegionTranslationOptions) +translator.register(RestrictionCodeType, RestrictionCodeTypeTranslationOptions) +translator.register(License, LicenseTranslationOptions) +translator.register(ResourceBase) diff --git a/geonode/base/urls.py b/geonode/base/urls.py new file mode 100644 index 0000000..a766a6d --- /dev/null +++ b/geonode/base/urls.py @@ -0,0 +1,75 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.urls import include, re_path + +from .views import ( + DatasetsAutocomplete, + RegionAutocomplete, + ThesaurusAvailable, + OwnerRightsRequestView, + ResourceBaseAutocomplete, + HierarchicalKeywordAutocomplete, + ThesaurusKeywordLabelAutocomplete, + LinkedResourcesAutocomplete, +) + + +urlpatterns = [ + re_path( + r"^autocomplete_response/$", + ResourceBaseAutocomplete.as_view(), + name="autocomplete_base", + ), + re_path( + r"^autocomplete_linked_resource/$", + LinkedResourcesAutocomplete.as_view(), + name="autocomplete_linked_resource", + ), + re_path( + r"^autocomplete_region/$", + RegionAutocomplete.as_view(), + name="autocomplete_region", + ), + re_path( + r"^autocomplete_hierachical_keyword/$", + HierarchicalKeywordAutocomplete.as_view(), + name="autocomplete_hierachical_keyword", + ), + re_path( + r"^thesaurus_available", + ThesaurusAvailable.as_view(), + name="thesaurus_available", + ), + re_path( + r"^thesaurus_autocomplete/$", + ThesaurusKeywordLabelAutocomplete.as_view(), + name="thesaurus_autocomplete", + ), + re_path( + r"^datasets_autocomplete/$", + DatasetsAutocomplete.as_view(), + name="datasets_autocomplete", + ), + re_path( + r"^resource_rights/(?P\d+)$", + OwnerRightsRequestView.as_view(), + name="owner_rights_request", + ), + re_path(r"^", include("geonode.base.api.urls")), +] diff --git a/geonode/base/utils.py b/geonode/base/utils.py new file mode 100644 index 0000000..0afee61 --- /dev/null +++ b/geonode/base/utils.py @@ -0,0 +1,215 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""Utilities for managing GeoNode base models +""" + +# Standard Modules +import re +import json +import logging +from schema import Schema +from dateutil.parser import isoparse +from datetime import datetime, timedelta + +# Django functionality +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError + +# Geonode functionality +from geonode.layers.models import Dataset +from geonode.base.models import ResourceBase, Link, Configuration +from geonode.security.utils import AdvancedSecurityWorkflowManager +from geonode.thumbs.utils import get_thumbs, remove_thumb +from geonode.utils import get_legend_url + +logger = logging.getLogger("geonode.base.utils") + +_names = [ + "Zipped Shapefile", + "Zipped", + "Shapefile", + "GML 2.0", + "GML 3.1.1", + "CSV", + "GeoJSON", + "Excel", + "Legend", + "GeoTIFF", + "GZIP", + "Original Dataset", + "ESRI Shapefile", + "View in Google Earth", + "KML", + "KMZ", + "Atom", + "DIF", + "Dublin Core", + "ebRIM", + "FGDC", + "ISO", + "ISO with XSL", +] + +thumb_filename_regex = re.compile( + r"^(document|map|layer|dataset|geoapp)-([a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12})-thumb-([a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12})\.png$" +) + + +def get_thumb_uuid(filename): + """Fetches the UUID associated with the given thumbnail file""" + result = thumb_filename_regex.search(filename) + uuid = result.group(2) if result else None + + return uuid + + +def delete_orphaned_thumbs(): + """ + Deletes orphaned thumbnails. + """ + deleted = [] + thumb_uuids = {get_thumb_uuid(filename): filename for filename in get_thumbs()} + db_uuids = ResourceBase.objects.filter(uuid__in=thumb_uuids.keys()).values_list("uuid", flat=True) + orphaned_uuids = set(thumb_uuids.keys()) - set(db_uuids) + orphaned_thumbs = (thumb_uuids[uuid] for uuid in orphaned_uuids if uuid is not None) + + for filename in orphaned_thumbs: + try: + remove_thumb(filename) + deleted.append(filename) + except NotImplementedError as e: + logger.error(f"Failed to delete orphaned thumbnail '{filename}': {e}") + + return deleted + + +def remove_duplicate_links(resource): + """ + Makes a scan of Links related to the resource and removes the duplicates. + It also regenerates the Legend link in case this is missing for some reason. + """ + for _n in _names: + _links = Link.objects.filter(resource__id=resource.id, name=_n) + _cnt = _links.count() + while _cnt > 1: + _links.last().delete() + _cnt -= 1 + + if isinstance(resource, Dataset): + # fixup Legend links + layer = resource + if layer.default_style and not layer.get_legend_url(style_name=layer.default_style.name): + Link.objects.update_or_create( + resource=layer.resourcebase_ptr, + name="Legend", + extension="png", + url=get_legend_url(layer, layer.default_style.name), + mime="image/png", + link_type="image", + ) + + +def configuration_session_cache(session): + CONFIG_CACHE_TIMEOUT_SEC = 60 + + _config = session.get("config") + _now = datetime.utcnow() + _dt = isoparse(_config.get("expiration")) if _config else _now + if _config is None or _dt < _now: + config = Configuration.load() + _dt = _now + timedelta(seconds=CONFIG_CACHE_TIMEOUT_SEC) + cached_config = {"configuration": {}, "expiration": _dt.isoformat()} + + for field_name in ["read_only", "maintenance"]: + cached_config["configuration"][field_name] = getattr(config, field_name) + + session["config"] = cached_config + + +class OwnerRightsRequestViewUtils: + @staticmethod + def get_message_recipients(owner): + User = get_user_model() + allowed_users = User.objects.none() + if AdvancedSecurityWorkflowManager.is_admin_moderate_mode(): + allowed_users |= User.objects.filter(is_superuser=True).exclude(pk=owner.pk) + try: + from geonode.groups.models import GroupProfile + + groups = owner.groups.all() + obj_group_managers = [] + for group in groups: + try: + group_profile = GroupProfile.objects.get(slug=group.name) + managers = group_profile.get_managers() + for manager in managers: + if manager not in obj_group_managers and not manager.is_superuser: + obj_group_managers.append(manager) + except GroupProfile.DoesNotExist: + pass + allowed_users |= User.objects.filter(id__in=[_u.id for _u in obj_group_managers]) + except Exception: + pass + + return allowed_users + + @staticmethod + def get_resource(resource_base): + return resource_base.get_real_instance() + + +def validate_extra_metadata(data, instance): + if not data: + return data + + # starting validation of extra metadata passed via JSON + # if schema for metadata validation is not defined, an error is raised + resource_type = instance.polymorphic_ctype.model if instance.polymorphic_ctype else instance.class_name.lower() + extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) + if not extra_metadata_validation_schema: + raise ValidationError(f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}") + # starting json structure validation. The Field can contain multiple metadata + try: + if isinstance(data, str): + data = json.loads(data) + except Exception: + raise ValidationError("The value provided for the Extra metadata field is not a valid JSON") + + # looping on all the single metadata provided. If it doen't match the schema an error is raised + for _index, _metadata in enumerate(data): + try: + Schema(extra_metadata_validation_schema).validate(_metadata) + except Exception as e: + raise ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") + # conerted because in this case, we can store a well formated json instead of the user input + return data + + +def remove_country_from_languagecode(language: str): + """Remove country code (us) from language name (en-us) + >>> remove_country_from_lanugecode("en-us") + 'en' + """ + if "-" not in language: + return language + + lang, _, _ = language.lower().partition("-") + return lang diff --git a/geonode/base/views.py b/geonode/base/views.py new file mode 100644 index 0000000..71c0571 --- /dev/null +++ b/geonode/base/views.py @@ -0,0 +1,809 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import json +import logging +import ast +import warnings +import traceback + +from dal import views, autocomplete +from user_messages.models import Message +from guardian.shortcuts import get_objects_for_user + +from django.conf import settings +from django.http import Http404 +from django.shortcuts import render +from django.http import HttpResponse +from django.views.generic import FormView +from django.core.exceptions import ObjectDoesNotExist +from django.views.decorators.clickjacking import xframe_options_sameorigin +from django.http import HttpResponseRedirect +from django.contrib.auth import get_user_model +from django.contrib import messages +from django.utils.translation import gettext_lazy as _ +from django.core.exceptions import PermissionDenied +from django.contrib.auth.decorators import login_required +from django.contrib.auth.mixins import LoginRequiredMixin +from django.urls import reverse +from django.utils.translation import get_language + +# Geonode dependencies +from geonode.maps.models import Map +from geonode.layers.models import Dataset +from geonode.utils import resolve_object +from geonode.base import register_event +from geonode.documents.models import Document +from geonode.groups.models import GroupProfile +from geonode.tasks.tasks import set_permissions +from geonode.resource.manager import resource_manager +from geonode.security.utils import get_visible_resources +from geonode.notifications_helper import send_notification +from geonode.base.utils import OwnerRightsRequestViewUtils, remove_country_from_languagecode +from geonode.base.forms import UserAndGroupPermissionsForm + +from geonode.base.forms import BatchEditForm, OwnerRightsRequestForm +from geonode.base.models import Region, ResourceBase, HierarchicalKeyword, ThesaurusKeyword, ThesaurusKeywordLabel + +from geonode.base.enumerations import SOURCE_TYPE_LOCAL + +from geonode.client.hooks import hookset +from geonode.people.forms import ProfileForm +from geonode.monitoring.models import EventType +from geonode.base.auth import get_or_create_token +from geonode.security.views import _perms_info_json +from geonode.security.utils import get_user_visible_groups +from geonode.decorators import check_keyword_write_perms + +from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm +from geonode.base.models import Thesaurus, TopicCategory + +from .forms import ResourceBaseForm + +logger = logging.getLogger(__name__) + + +def get_url_for_app_model(model, model_class): + return reverse(f"admin:{model_class._meta.app_label}_{model}_changelist") + # was: return f'/admin/{model_class._meta.app_label}/{model}/' + + +def get_url_for_model(model): + url = f"admin:{model.lower()}s_{model.lower()}_changelist" + if model.lower() == "dataset": + url = f"admin:layers_{model.lower()}_changelist" + return reverse(url) + # was: f'/admin/{model.lower()}s/{model.lower()}/' + + +def user_and_group_permission(request, model): + if not request.user.is_superuser: + raise PermissionDenied + + model_mapper = {"profile": get_user_model(), "groupprofile": GroupProfile} + + model_class = model_mapper[model] + + ids = request.POST.get("ids") + if "cancel" in request.POST or not ids: + return HttpResponseRedirect(get_url_for_app_model(model, model_class)) + + users_usernames = None + groups_names = None + + if request.method == "POST": + form = UserAndGroupPermissionsForm(request.POST) + ids = ids.split(",") + _message = "" + _errors = False + if form.is_valid(): + resources_names = form.cleaned_data.get("layers") + users_usernames = ( + [user.username for user in model_class.objects.filter(id__in=ids)] if model == "profile" else None + ) + groups_names = ( + [group_profile.group.name for group_profile in model_class.objects.filter(id__in=ids)] + if model in ("group", "groupprofile") + else None + ) + + if ( + users_usernames + and "AnonymousUser" in users_usernames + and (not groups_names or "anonymous" not in groups_names) + ): + if not groups_names: + groups_names = [] + groups_names.append("anonymous") + if ( + groups_names + and "anonymous" in groups_names + and (not users_usernames or "AnonymousUser" not in users_usernames) + ): + if not users_usernames: + users_usernames = [] + users_usernames.append("AnonymousUser") + + delete_flag = form.cleaned_data.get("mode") == "unset" + permissions_names = form.cleaned_data.get("permission_type") + + if permissions_names: + if "edit" in permissions_names and users_usernames and "AnonymousUser" in users_usernames: + if not _errors: + _message = '"EDIT" permissions not allowed for the "AnonymousUser".' + _errors = True + else: + set_permissions.apply_async( + args=([permissions_names], resources_names, users_usernames, groups_names, delete_flag), + expiration=30, + ) + if not _errors: + _message = f'The asyncronous permissions {form.cleaned_data.get("mode")} request for {", ".join(users_usernames or groups_names)} has been sent' + else: + if not _errors: + _message = "No permissions have been set." + _errors = True + else: + if not _errors: + _message = f"Some error has occured {form.errors}" + _errors = True + + if "layers" in form.errors and "invalid_choice" in (x.code for x in form.errors["layers"].data): + _invalid_dataset_id = [] + for el in form.errors["layers"]: + _invalid_dataset_id.extend([x for x in el.split(" ") if x.isnumeric()]) + _message = f"The following dataset ID selected are not part of the available choices: {','.join(_invalid_dataset_id)}. They are probably in a dirty state, Please try again later" + + messages.add_message(request, (messages.INFO if not _errors else messages.ERROR), _message) + return HttpResponseRedirect(get_url_for_app_model(model, model_class)) + + form = UserAndGroupPermissionsForm( + { + "permission_type": "view", + "mode": "set", + } + ) + return render(request, "base/user_and_group_permissions.html", context={"form": form, "model": model}) + + +def batch_modify(request, model): + if not request.user.is_superuser: + raise PermissionDenied + if model == "Document": + Resource = Document + if model == "Dataset": + Resource = Dataset + if model == "Map": + Resource = Map + template = "base/batch_edit.html" + ids = request.POST.get("ids") + + if "cancel" in request.POST or not ids: + return HttpResponseRedirect(get_url_for_model(model)) + + if request.method == "POST": + form = BatchEditForm(request.POST) + if form.is_valid(): + keywords = [keyword.strip() for keyword in form.cleaned_data.pop("keywords").split(",") if keyword] + regions = form.cleaned_data.pop("regions") + ids = form.cleaned_data.pop("ids") + if not form.cleaned_data.get("date"): + form.cleaned_data.pop("date") + + to_update = {} + for _key, _value in form.cleaned_data.items(): + if _value: + to_update[_key] = _value + resources = Resource.objects.filter(id__in=ids.split(",")) + resources.update(**to_update) + if regions: + regions_through = Resource.regions.through + new_regions = [regions_through(region=regions, resourcebase=resource) for resource in resources] + regions_through.objects.bulk_create(new_regions, ignore_conflicts=True) + + if keywords: + keywords_through = Resource.keywords.through + keywords_through.objects.filter(content_object__in=resources).delete() + + def get_or_create(keyword): + try: + return HierarchicalKeyword.objects.get(name=keyword) + except HierarchicalKeyword.DoesNotExist: + return HierarchicalKeyword.add_root(name=keyword) + + hierarchical_keyword = [get_or_create(keyword) for keyword in keywords] + + new_keywords = [] + for keyword in hierarchical_keyword: + new_keywords += [ + keywords_through(content_object=resource, tag_id=keyword.pk) for resource in resources + ] + keywords_through.objects.bulk_create(new_keywords, ignore_conflicts=True) + + return HttpResponseRedirect(get_url_for_model(model)) + + return render( + request, + template, + context={ + "form": form, + "ids": ids, + "model": model, + }, + ) + + form = BatchEditForm() + return render( + request, + template, + context={ + "form": form, + "ids": ids, + "model": model, + }, + ) + + +class SimpleSelect2View(autocomplete.Select2QuerySetView): + """Generic select2 view for autocompletes + Params: + model: model to perform the autocomplete query on + filter_arg: property to filter with ie. name__icontains + """ + + def __init__(self, *args, **kwargs): + super(views.BaseQuerySetView, self).__init__(*args, **kwargs) + if not hasattr(self, "filter_arg"): + raise AttributeError("SimpleSelect2View missing required 'filter_arg' argument") + + def get_queryset(self): + qs = super(views.BaseQuerySetView, self).get_queryset().order_by("pk") + + if self.q: + qs = qs.filter(**{self.filter_arg: self.q}) + return qs + + +class ResourceBaseAutocomplete(autocomplete.Select2QuerySetView): + """Base resource autocomplete - searches all the resources by title + returns any visible resources in this queryset for autocomplete + """ + + def get_queryset(self): + request = self.request + + permitted = get_objects_for_user(request.user, "base.view_resourcebase") + qs = ResourceBase.objects.all().filter(id__in=permitted) + + if self.q: + qs = qs.filter(title__icontains=self.q).order_by("title") + + return get_visible_resources( + qs, + request.user if request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + )[:100] + + +class LinkedResourcesAutocomplete(autocomplete.Select2QuerySetView): + def get_queryset(self): + qs = ResourceBase.objects.order_by("title") + + if self.q: + qs = qs.filter(title__icontains=self.q) + + if self.forwarded and "exclude" in self.forwarded: + qs = qs.exclude(pk=self.forwarded["exclude"]) + + return get_visible_resources( + qs, + self.request.user if self.request else None, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + def get_result_label(self, result): + return f"{result.title} [{_(result.polymorphic_ctype.model)}]" + + +class RegionAutocomplete(SimpleSelect2View): + model = Region + filter_arg = "name__icontains" + + +class HierarchicalKeywordAutocomplete(SimpleSelect2View): + model = HierarchicalKeyword + filter_arg = "slug__icontains" + + +class ThesaurusKeywordLabelAutocomplete(autocomplete.Select2QuerySetView): + def get_queryset(self): + thesaurus = settings.THESAURUS + tname = thesaurus["name"] + lang = "en" + + # Filters thesaurus results based on thesaurus name and language + qs = ThesaurusKeywordLabel.objects.all().filter(keyword__thesaurus__identifier=tname, lang=lang) + + if self.q: + qs = qs.filter(label__icontains=self.q) + + return qs + + # Overides the get results method to return custom json to frontend + def get_results(self, context): + return [ + { + "id": self.get_result_value(result.keyword), + "text": self.get_result_label(result), + "selected_text": self.get_selected_result_label(result), + } + for result in context["object_list"] + ] + + +class DatasetsAutocomplete(SimpleSelect2View): + model = Dataset + filter_arg = "title__icontains" + + def get_results(self, context): + return [ + { + "id": self.get_result_value(result), + "text": self.get_result_label(result.title), + "selected_text": self.get_selected_result_label(result.title), + } + for result in context["object_list"] + ] + + +class ThesaurusAvailable(autocomplete.Select2QuerySetView): + def get_queryset(self): + tid = self.request.GET.get("sysid") + lang = get_language() + keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=tid) + + # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") + if len(qs_keyword_ids) == 0: + lang = remove_country_from_languagecode(lang) + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter( + lang=lang, keyword_id__in=keyword_id_for_given_thesaurus + ).values("keyword_id") + + not_qs_ids = ( + ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids) + .order_by("keyword_id") + .distinct("keyword_id") + .values("keyword_id") + ) + qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus) + if self.q: + qs = qs.filter(label__istartswith=self.q) + + qs_local = list(qs) + qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids)) + return qs_local + qs_non_local + + def get_results(self, context): + return [ + { + "id": str(result.keyword.pk) if isinstance(result, ThesaurusKeywordLabel) else str(result.pk), + "text": self.get_result_label(result), + "selected_text": self.get_selected_result_label(result), + } + for result in context["object_list"] + ] + + +class OwnerRightsRequestView(LoginRequiredMixin, FormView): + template_name = "owner_rights_request.html" + form_class = OwnerRightsRequestForm + resource = None + redirect_field_name = "next" + + def get_success_url(self): + return self.resource.get_absolute_url() + + def get(self, request, *args, **kwargs): + r_base = ResourceBase.objects.get(pk=kwargs.get("pk")) + self.resource = OwnerRightsRequestViewUtils.get_resource(r_base) + initial = {"resource": r_base} + form = self.form_class(initial=initial) + return render(request, self.template_name, {"form": form, "resource": self.resource}) + + def post(self, request, *args, **kwargs): + r_base = ResourceBase.objects.get(pk=kwargs.get("pk")) + self.resource = OwnerRightsRequestViewUtils.get_resource(r_base) + form = self.form_class(request.POST) + if form.is_valid(): + reason = form.cleaned_data["reason"] + notice_type_label = "request_resource_edit" + recipients = OwnerRightsRequestViewUtils.get_message_recipients(self.resource.owner) + + Message.objects.new_message( + from_user=request.user, + to_users=recipients, + subject=_("System message: A request to modify resource"), + content=_("The resource owner has requested to modify the resource") + ".\n" + " " + _("Resource title") + ": " + self.resource.title + ".\n" + " " + + _("Reason for the request") + + ': "' + + reason + + '".\n' + + " " + + _( + 'To allow the change, set the resource to not "Approved" under the metadata settings' + + "and write message to the owner to notify him" + ) + + ".", + ) + send_notification( + recipients, + notice_type_label, + {"resource": self.resource, "site_url": settings.SITEURL[:-1], "reason": reason}, + ) + return self.form_valid(form) + else: + return self.form_invalid(form) + + +logger = logging.getLogger("geonode.base.metadata") + +_PERMISSION_MSG_GENERIC = _("You do not have permissions for this resource.") +_PERMISSION_MSG_METADATA = _("You are not allowed to modify this resource's metadata.") +_PERMISSION_MSG_VIEW = _("You are not allowed to view this resource.") + + +def _resolve_resourcebase(request, id, permission="base.change_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs): + """ + Resolve the resourcebase by the provided typename and check the optional permission. + """ + + return resolve_object(request, ResourceBase, {"pk": id}, permission=permission, permission_msg=msg, **kwargs) + + +@xframe_options_sameorigin +def resourcebase_embed(request, resourcebaseid, template="base/base_edit.html"): + """ + The view that returns the app composer opened to + the app with the given app ID. + """ + try: + resourcebase_obj = _resolve_resourcebase( + request, resourcebaseid, "base.view_resourcebase", _PERMISSION_MSG_VIEW + ) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not resourcebase_obj: + raise Http404(_("Not found")) + + # Call this first in order to be sure "perms_list" is correct + permissions_json = _perms_info_json(resourcebase_obj) + + perms_list = resourcebase_obj.get_user_perms(request.user) + + group = None + if resourcebase_obj.group: + try: + group = GroupProfile.objects.get(slug=resourcebase_obj.group.name) + except GroupProfile.DoesNotExist: + group = None + + r = resourcebase_obj + if request.method in ("POST", "PATCH", "PUT"): + r = resource_manager.update(resourcebase_obj.uuid, instance=resourcebase_obj, notify=True) + + resource_manager.set_permissions( + resourcebase_obj.uuid, instance=resourcebase_obj, permissions=ast.literal_eval(permissions_json) + ) + + resource_manager.set_thumbnail(resourcebase_obj.uuid, instance=resourcebase_obj, overwrite=False) + + access_token = None + if request and request.user: + access_token = get_or_create_token(request.user) + if access_token and not access_token.is_expired(): + access_token = access_token.token + else: + access_token = None + + _config = json.dumps(r.blob) + _ctx = { + "appId": resourcebaseid, + "appType": resourcebase_obj.resource_type, + "config": _config, + "user": request.user, + "access_token": access_token, + "resource": resourcebase_obj, + "group": group, + "perms_list": perms_list, + "permissions_json": permissions_json, + "preview": getattr(settings, "GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore"), + } + + return render(request, template, context=_ctx) + + +def resourcebase_metadata_detail( + request, resourcebaseid, template="base/base_metadata_detail.html", custom_metadata=None +): + try: + resourcebase_obj = _resolve_resourcebase(request, resourcebaseid, "view_resourcebase", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not resourcebase_obj: + raise Http404(_("Not found")) + + group = None + if resourcebase_obj.group: + try: + group = GroupProfile.objects.get(slug=resourcebase_obj.group.name) + except ObjectDoesNotExist: + group = None + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + register_event(request, EventType.EVENT_VIEW_METADATA, resourcebase_obj) + + return render( + request, + template, + context={ + "resource": resourcebase_obj, + "group": group, + "SITEURL": site_url, + "custom_metadata": custom_metadata, + }, + ) + + +@login_required +@check_keyword_write_perms +def resourcebase_metadata( + request, + resourcebaseid, + template="base/base_metadata.html", + ajax=True, + panel_template="base/base_panels.html", + custom_metadata=None, +): + resourcebase_obj = None + try: + resourcebase_obj = _resolve_resourcebase( + request, resourcebaseid, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA + ) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not resourcebase_obj: + raise Http404(_("Not found")) + + # Add metadata_author or poc if missing + resourcebase_obj.add_missing_metadata_author_or_poc() + resource_type = resourcebase_obj.resource_type + topic_category = resourcebase_obj.category + subtype = resourcebase_obj.subtype + current_keywords = [keyword.name for keyword in resourcebase_obj.keywords.all()] + + topic_thesaurus = resourcebase_obj.tkeywords.all() + + if request.method == "POST": + resourcebase_form = ResourceBaseForm( + request.POST, instance=resourcebase_obj, prefix="resource", user=request.user + ) + category_form = CategoryForm( + request.POST, + prefix="category_choice_field", + initial=( + int(request.POST["category_choice_field"]) + if "category_choice_field" in request.POST and request.POST["category_choice_field"] + else None + ), + ) + + if hasattr(settings, "THESAURUS"): + tkeywords_form = TKeywordForm(request.POST) + else: + tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords") + + else: + resourcebase_form = ResourceBaseForm(instance=resourcebase_obj, prefix="resource", user=request.user) + resourcebase_form.disable_keywords_widget_for_non_superuser(request.user) + category_form = CategoryForm( + prefix="category_choice_field", initial=topic_category.id if topic_category else None + ) + + # Create THESAURUS widgets + lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, "THESAURUS_DEFAULT_LANG") else "en" + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + warnings.warn( + "The settings for Thesaurus has been moved to Model, \ + this feature will be removed in next releases", + DeprecationWarning, + ) + dataset_tkeywords = resourcebase_obj.tkeywords.all() + tkeywords_list = "" + if dataset_tkeywords and len(dataset_tkeywords) > 0: + tkeywords_ids = dataset_tkeywords.values_list("id", flat=True) + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + el = settings.THESAURUS + thesaurus_name = el["name"] + try: + t = Thesaurus.objects.get(identifier=thesaurus_name) + for tk in t.thesaurus.filter(pk__in=tkeywords_ids): + tkl = tk.keyword.filter(lang=lang) + if len(tkl) > 0: + tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True))) + tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids + except Exception: + tb = traceback.format_exc() + logger.error(tb) + tkeywords_form = TKeywordForm(instance=resourcebase_obj) + else: + tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords") + # set initial values for thesaurus form + for tid in tkeywords_form.fields: + values = [] + values = [keyword.id for keyword in topic_thesaurus if int(tid) == keyword.thesaurus.id] + tkeywords_form.fields[tid].initial = values + + if ( + request.method == "POST" + and resourcebase_form.is_valid() + and category_form.is_valid() + and tkeywords_form.is_valid() + ): + new_keywords = current_keywords if request.keyword_readonly else resourcebase_form.cleaned_data.pop("keywords") + new_regions = resourcebase_form.cleaned_data.pop("regions") + + new_category = None + if ( + category_form + and "category_choice_field" in category_form.cleaned_data + and category_form.cleaned_data["category_choice_field"] + ): + new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"])) + resourcebase_form.cleaned_data.pop("ptype") + + resourcebase_obj = resourcebase_form.instance + # update contact roles + resourcebase_obj.set_contact_roles_from_metadata_edit(resourcebase_form) + + vals = dict(category=new_category, subtype=subtype) + + resourcebase_form.cleaned_data.pop("metadata") + extra_metadata = resourcebase_form.cleaned_data.pop("extra_metadata") + + resourcebase_form.save_linked_resources() + resourcebase_form.cleaned_data.pop("linked_resources") + + vals.update({"resource_type": resource_type, "sourcetype": SOURCE_TYPE_LOCAL}) + + register_event(request, EventType.EVENT_CHANGE_METADATA, resourcebase_obj) + if not ajax: + return HttpResponseRedirect(hookset.resourcebase_detail_url(resourcebase_obj)) + + message = resourcebase_obj.id + + try: + # Keywords from THESAURUS management + # Rewritten to work with updated autocomplete + if not tkeywords_form.is_valid(): + return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400)) + + thesaurus_setting = getattr(settings, "THESAURUS", None) + if thesaurus_setting: + tkeywords_data = tkeywords_form.cleaned_data["tkeywords"] + tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"]) + resourcebase_obj.tkeywords.set(tkeywords_data) + elif Thesaurus.objects.all().exists(): + fields = tkeywords_form.cleaned_data + resourcebase_obj.tkeywords.set(tkeywords_form.cleanx(fields)) + + except Exception: + tb = traceback.format_exc() + logger.error(tb) + + if "group" in resourcebase_form.changed_data: + vals["group"] = resourcebase_form.cleaned_data.get("group") + if any([x in resourcebase_form.changed_data for x in ["is_approved", "is_published"]]): + vals["is_approved"] = resourcebase_form.cleaned_data.get("is_approved", resourcebase_obj.is_approved) + vals["is_published"] = resourcebase_form.cleaned_data.get("is_published", resourcebase_obj.is_published) + else: + vals.pop("is_approved", None) + vals.pop("is_published", None) + + resource_manager.update( + resourcebase_obj.uuid, + instance=resourcebase_obj, + keywords=new_keywords, + regions=new_regions, + notify=True, + vals=vals, + extra_metadata=json.loads(extra_metadata), + ) + + resource_manager.set_thumbnail(resourcebase_obj.uuid, instance=resourcebase_obj, overwrite=False) + + return HttpResponse(json.dumps({"message": message})) + elif request.method == "POST" and ( + not resourcebase_form.is_valid() or not category_form.is_valid() or not tkeywords_form.is_valid() + ): + errors_list = { + **resourcebase_form.errors.as_data(), + **category_form.errors.as_data(), + **tkeywords_form.errors.as_data(), + } + logger.error(f"resourcebase Metadata form is not valid: {errors_list}") + out = {"success": False, "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()]} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + # - POST Request Ends here - + + # define contact role forms + contact_role_forms_context = {} + for role in resourcebase_obj.get_multivalue_role_property_names(): + resourcebase_form.fields[role].initial = [p.username for p in resourcebase_obj.__getattribute__(role)] + role_form = ProfileForm(prefix=role) + role_form.hidden = True + contact_role_forms_context[f"{role}_form"] = role_form + + metadata_author_groups = get_user_visible_groups(request.user) + + if not request.user.can_publish(resourcebase_obj): + resourcebase_form.fields["is_published"].widget.attrs.update({"disabled": "true"}) + if not request.user.can_approve(resourcebase_obj): + resourcebase_form.fields["is_approved"].widget.attrs.update({"disabled": "true"}) + + register_event(request, EventType.EVENT_VIEW_METADATA, resourcebase_obj) + return render( + request, + template, + context={ + "resource": resourcebase_obj, + "resourcebase": resourcebase_obj, + "panel_template": panel_template, + "custom_metadata": custom_metadata, + "resourcebase_form": resourcebase_form, + "category_form": category_form, + "tkeywords_form": tkeywords_form, + "metadata_author_groups": metadata_author_groups, + "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False), + "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + "UI_MANDATORY_FIELDS": list( + set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", [])) + | set(getattr(settings, "UI_REQUIRED_FIELDS", [])) + ), + **contact_role_forms_context, + "UI_ROLES_IN_TOGGLE_VIEW": resourcebase_obj.get_ui_toggled_role_property_names(), + }, + ) + + +@login_required +def resourcebase_metadata_advanced(request, resourcebaseid): + return resourcebase_metadata(request, resourcebaseid, template="base/base_metadata_advanced.html") diff --git a/geonode/base/widgets.py b/geonode/base/widgets.py new file mode 100644 index 0000000..a4e32bb --- /dev/null +++ b/geonode/base/widgets.py @@ -0,0 +1,40 @@ +from typing import List + +from dal_select2_taggit.widgets import TaggitSelect2 +from django.http.request import QueryDict + + +class TaggitSelect2Custom(TaggitSelect2): + """Overriding Select2 tag widget for taggit's TagField. + Fixes error in tests where 'value' is None. + """ + + def value_from_datadict(self, data, files, name): + """Handle multi-word tag. + + Insure there's a comma when there's only a single multi-word tag, + or tag "Multi word" would end up as "Multi" and "word". + """ + + try: + value = super().value_from_datadict(data, files, name) + if value and "," not in value: + value = f"{value}," + return value + except TypeError: + return "" + + +class TaggitProfileSelect2Custom(TaggitSelect2): + """Overriding Select2 tag widget for ContactRoleField.""" + + def value_from_datadict(self, data, files, name) -> List[str]: + """Handle multi-profiles. + + returns list of selected elements + """ + if type(data) is dict and name in data: + return data[name] + elif type(data) is QueryDict: + return data.getlist(name) + return [] diff --git a/geonode/br/__init__.py b/geonode/br/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/br/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/br/admin.py b/geonode/br/admin.py new file mode 100644 index 0000000..056364f --- /dev/null +++ b/geonode/br/admin.py @@ -0,0 +1,37 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib import admin +from geonode.br.models import RestoredBackup + + +@admin.register(RestoredBackup) +class RestoredBackupAdmin(admin.ModelAdmin): + readonly_fields = ("name", "restoration_date", "archive_md5", "creation_date") + actions = None + list_display = ("name", "restoration_date", "archive_md5", "creation_date") + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False diff --git a/geonode/br/apps.py b/geonode/br/apps.py new file mode 100644 index 0000000..55624e0 --- /dev/null +++ b/geonode/br/apps.py @@ -0,0 +1,25 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.apps import AppConfig + + +class BackupRestoreAppConfig(AppConfig): + name = "geonode.br" + verbose_name = "Backup/Restore" diff --git a/geonode/br/management/__init__.py b/geonode/br/management/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/br/management/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/br/management/commands/__init__.py b/geonode/br/management/commands/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/br/management/commands/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/br/management/commands/backup.py b/geonode/br/management/commands/backup.py new file mode 100644 index 0000000..c54c040 --- /dev/null +++ b/geonode/br/management/commands/backup.py @@ -0,0 +1,466 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import os +import time +import shutil +import requests +import re +import logging + +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import LocalAsset + +from .utils import utils + +from requests.auth import HTTPBasicAuth +from xmltodict import parse as parse_xml +from urllib.parse import urlparse, urljoin + +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand, CommandError + +from geonode.utils import DisableDjangoSignals, get_dir_time_suffix, zip_dir, copy_tree + +from geonode.base.models import Configuration + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Backup the GeoNode application data" + + def add_arguments(self, parser): + # Named (optional) arguments + utils.option(parser) + + utils.geoserver_option_list(parser) + + parser.add_argument( + "-i", + "--ignore-errors", + action="store_true", + dest="ignore_errors", + default=False, + help="Stop after any errors are encountered.", + ) + + parser.add_argument( + "-f", + "--force", + action="store_true", + dest="force_exec", + default=False, + help="Forces the execution without asking for confirmation.", + ) + + parser.add_argument("--skip-geoserver", action="store_true", default=False, help="Skips geoserver backup") + + parser.add_argument( + "--backup-dir", + dest="backup_dir", + help="Destination folder where to store the backup archive. It must be writable.", + ) + + parser.add_argument( + "--skip-read-only", + action="store_true", + dest="skip_read_only", + default=False, + help="Skips activation of the Read Only mode in backup procedure execution.", + ) + + parser.add_argument( + "--skip-logger-setup", + action="store_false", + dest="setup_logger", + help='Skips setup of the "geonode.br" logger, "br" handler and "br" format if not present in settings', + ) + + def handle(self, **options): + skip_read_only = options.get("skip_read_only") + config = Configuration.load() + + # activate read only mode and store it's original config value + if not skip_read_only: + original_read_only_value = config.read_only + config.read_only = True + config.save() + + if options.get("setup_logger"): + utils.setup_logger() + + try: + # execute backup procedure + self.execute_backup(**options) + except Exception: + raise + finally: + # restore read only mode's original value + if not skip_read_only: + config.read_only = original_read_only_value + config.save() + + def execute_backup(self, **options): + ignore_errors = options.get("ignore_errors") + config = utils.Config(options) + force_exec = options.get("force_exec") + backup_dir = options.get("backup_dir") + skip_geoserver = options.get("skip_geoserver") + + if not backup_dir or len(backup_dir) == 0: + raise CommandError("Destination folder '--backup-dir' is mandatory") + + if not force_exec: + print("Before proceeding with the Backup, please ensure that:") + print(" 1. The backend (DB or whatever) is accessible and you have rights") + print(" 2. The GeoServer is up and running and reachable from this machine") + + message = "You want to proceed?" + + if force_exec or utils.confirm(prompt=message, resp=False): + # Create Target Folder + dir_time_suffix = get_dir_time_suffix() + target_folder = os.path.join(backup_dir, dir_time_suffix) + if not os.path.exists(target_folder): + os.makedirs(target_folder, exist_ok=True) + # Temporary folder to store backup files. It will be deleted at the end. + os.chmod(target_folder, 0o777) + + if not skip_geoserver: + self.create_geoserver_backup(config, settings, target_folder, ignore_errors) + self.dump_geoserver_raster_data(config, settings, target_folder) + self.dump_geoserver_vector_data(config, settings, target_folder) + self.dump_geoserver_externals(config, settings, target_folder) + else: + logger.info("Skipping geoserver backup") + + # Deactivate GeoNode Signals + with DisableDjangoSignals(): + # Dump Fixtures + logger.info("*** Dumping GeoNode fixtures...") + + fixtures_target = os.path.join(target_folder, "fixtures") + os.makedirs(fixtures_target, exist_ok=True) + + for app_name, dump_name in zip(config.app_names, config.dump_names): + # prevent dumping BackupRestore application + if app_name == "br": + continue + + logger.info(f" - Dumping '{app_name}' into '{dump_name}.json'") + # Point stdout at a file for dumping data to. + output_file = os.path.join(fixtures_target, f"{dump_name}.json") + call_command("dumpdata", app_name, output=output_file) + + # Store Media Root + + media_folder = os.path.join(target_folder, utils.MEDIA_ROOT) + logger.info("*** Dumping GeoNode media folder...") + self.backup_folder(folder=media_folder, root=settings.MEDIA_ROOT, config=config) + + logger.info("*** Dumping GeoNode assets folder...") + assets_folder = os.path.join(target_folder, utils.ASSETS_ROOT) + self.backup_folder(folder=assets_folder, root=settings.ASSETS_ROOT, config=config) + for instance in LocalAsset.objects.iterator(): + if not LocalAssetHandler._are_files_managed(instance): + logger.warning(f"The file for the asset with id {instance.pk} were not backup since is not managed by GeoNode") + + # Create Final ZIP Archive + logger.info("*** Creating final ZIP archive...") + + backup_archive = os.path.join(backup_dir, f"{dir_time_suffix}.zip") + logger.info(f"Creating zip {backup_archive}...") + + zip_dir(target_folder, backup_archive) + + # Generate a md5 hash of a backup archive and save it + backup_md5_file = os.path.join(backup_dir, f"{dir_time_suffix}.md5") + zip_archive_md5 = utils.md5_file_hash(backup_archive) + with open(backup_md5_file, "w") as md5_file: + md5_file.write(zip_archive_md5) + + # Generate the ini file with the current settings used by the backup command + backup_ini_file = os.path.join(backup_dir, f"{dir_time_suffix}.ini") + with open(backup_ini_file, "w") as configfile: + config.config_parser.write(configfile) + + # Clean-up Temp Folder + logger.info("*** Final cleanup...") + try: + shutil.rmtree(target_folder) + except Exception: + logger.warning(f"WARNING: Could not be possible to delete the temp folder: '{target_folder}'") + + logger.info("Backup Finished. Archive generated.") + + return str(os.path.join(backup_dir, f"{dir_time_suffix}.zip")) + + def backup_folder(self, folder, root, config): + if not os.path.exists(folder): + os.makedirs(root, exist_ok=True) + + copy_tree( + root, + folder, + ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]), + ) + logger.info(f"Saved files from '{root}'") + + def create_geoserver_backup(self, config, settings, target_folder, ignore_errors): + # Create GeoServer Backup + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] + geoserver_bk_file = os.path.join(target_folder, "geoserver_catalog.zip") + + logger.info(f"*** Dumping GeoServer catalog [{url}] into '{geoserver_bk_file}'") + r = requests.put(f"{url}rest/reset/", auth=HTTPBasicAuth(user, passwd)) + if r.status_code != 200: + raise ValueError("Could not reset GeoServer catalog!") + r = requests.put(f"{url}rest/reload/", auth=HTTPBasicAuth(user, passwd)) + if r.status_code != 200: + raise ValueError("Could not reload GeoServer catalog!") + + error_backup = "Could not successfully backup GeoServer catalog [{}rest/br/backup/]: {} - {}" + + _options = [ + "BK_CLEANUP_TEMP=true", + "BK_SKIP_SETTINGS=false", + "BK_SKIP_SECURITY=false", + f'BK_BEST_EFFORT={("true" if ignore_errors else "false")}', + f"exclude.file.path={config.gs_exclude_file_path}", + ] + data = {"backup": {"archiveFile": geoserver_bk_file, "overwrite": "true", "options": {"option": _options}}} + headers = {"Accept": "application/json", "Content-type": "application/json"} + r = requests.post( + f"{url}rest/br/backup/", data=json.dumps(data), headers=headers, auth=HTTPBasicAuth(user, passwd) + ) + + if r.status_code in (200, 201, 406): + try: + r = requests.get( + f"{url}rest/br/backup.json", headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10 + ) + if r.status_code == 200: + gs_backup = r.json() + _url = urlparse(gs_backup["backups"]["backup"][-1]["href"]) + _url = f"{urljoin(url, _url.path)}?{_url.query}" + r = requests.get(_url, headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) + if r.status_code == 200: + gs_backup = r.json() + + if r.status_code != 200: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + except ValueError: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + gs_bk_exec_id = gs_backup["backup"]["execution"]["id"] + r = requests.get( + f"{url}rest/br/backup/{gs_bk_exec_id}.json", + headers=headers, + auth=HTTPBasicAuth(user, passwd), + timeout=10, + ) + if r.status_code == 200: + gs_bk_exec_status = gs_backup["backup"]["execution"]["status"] + gs_bk_exec_progress = gs_backup["backup"]["execution"]["progress"] + gs_bk_exec_progress_updated = "0/0" + while gs_bk_exec_status != "COMPLETED" and gs_bk_exec_status != "FAILED": + if gs_bk_exec_progress != gs_bk_exec_progress_updated: + gs_bk_exec_progress_updated = gs_bk_exec_progress + r = requests.get( + f"{url}rest/br/backup/{gs_bk_exec_id}.json", + headers=headers, + auth=HTTPBasicAuth(user, passwd), + timeout=10, + ) + if r.status_code == 200: + try: + gs_backup = r.json() + except ValueError: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + gs_bk_exec_status = gs_backup["backup"]["execution"]["status"] + gs_bk_exec_progress = gs_backup["backup"]["execution"]["progress"] + logger.info(f"Async backup status: {gs_bk_exec_status} - {gs_bk_exec_progress}") + time.sleep(3) + else: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + if gs_bk_exec_status == "FAILED": + raise ValueError(error_backup.format(url, r.status_code, r.text)) + _permissions = 0o777 + try: + os.chmod(geoserver_bk_file, _permissions) + status = os.stat(geoserver_bk_file) + if oct(status.st_mode & 0o777) != str(oct(_permissions)): + raise Exception(f"Could not update permissions of {geoserver_bk_file}") + except Exception as e: + logger.warning(e) + else: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + def dump_geoserver_raster_data(self, config, settings, target_folder): + if config.gs_data_dir and config.gs_dump_raster_data: + logger.info("*** Dump GeoServer raster data") + + for source_root, dest_folder in ( + ( + os.path.join(config.gs_data_dir, "geonode"), # Dump '$config.gs_data_dir/geonode' + os.path.join(target_folder, "gs_data_dir", "geonode"), + ), + ( + os.path.join(config.gs_data_dir, "data", "geonode"), # Dump '$config.gs_data_dir/data/geonode' + os.path.join(target_folder, "gs_data_dir", "data", "geonode"), + ), + ): + if not os.path.isabs(source_root): + source_root = os.path.join(settings.PROJECT_ROOT, "..", source_root) + logger.info(f"Dumping raster data from '{source_root}'...") + if os.path.exists(source_root): + if not os.path.exists(dest_folder): + os.makedirs(dest_folder, exist_ok=True) + copy_tree( + source_root, + dest_folder, + ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]), + ) + logger.info(f"Dumped raster data from '{source_root}'") + else: + logger.info(f"Skipped raster data directory '{source_root}' because it does not exist") + + def dump_geoserver_vector_data(self, config, settings, target_folder): + if config.gs_dump_vector_data: + logger.info("*** Dump GeoServer vector data") + + # Dump Vectorial Data from DB + datastore_name = settings.OGC_SERVER["default"]["DATASTORE"] + if datastore_name: + datastore = settings.DATABASES[datastore_name] + + gs_data_folder = os.path.join(target_folder, "gs_data_dir", "geonode") + if not os.path.exists(gs_data_folder): + os.makedirs(gs_data_folder, exist_ok=True) + + utils.dump_db( + config, + datastore["NAME"], + datastore["USER"], + datastore["PORT"], + datastore["HOST"], + datastore["PASSWORD"], + gs_data_folder, + ) + + def dump_geoserver_externals(self, config, settings, target_folder): + """Scan layers xml and see if there are external references. + + Find references to data outside data dir and include them in + backup. Also, some references may point to specific url, which + may not be available later. + """ + logger.info("*** Dumping GeoServer external resources") + + external_folder = os.path.join(target_folder, utils.EXTERNAL_ROOT) + + def copy_external_resource(abspath): + external_path = os.path.join(external_folder, abspath[1:]) + external_dir = os.path.dirname(external_path) + + if not os.path.isdir(external_dir): + os.makedirs(external_dir, exist_ok=True) + + try: + if not os.path.isdir(external_path) and os.path.exists(external_path): + shutil.copy2(abspath, external_path) + except shutil.SameFileError: + logger.warning(f"WARNING: {abspath} and {external_path} are the same file!") + + def match_filename(key, text, regexp=re.compile("^(.+)$")): + if key in ("filename",): + match = regexp.match(text.decode("utf-8")) + if match: + relpath = match.group(1) + try: + abspath = ( + relpath + if os.path.isabs(relpath) + else os.path.abspath(os.path.join(os.path.dirname(path), relpath)) + ) + if os.path.exists(abspath): + return abspath + except Exception: + logger.warning(f"WARNING: Error while trying to dump {text}") + return + + def match_fileurl(key, text, regexp=re.compile("^file:(.+)$")): + if key in ("url",): + match = regexp.match(text.decode("utf-8")) + if match: + relpath = match.group(1) + try: + abspath = ( + relpath + if os.path.isabs(relpath) + else os.path.abspath(os.path.join(config.gs_data_dir, relpath)) + ) + if os.path.exists(abspath): + return abspath + except Exception: + logger.warning(f"WARNING: Error while trying to dump {text}") + return + + def dump_external_resources_from_xml(path): + def find_external(tree, key=None): + if isinstance(tree, dict): + for key, value in tree.items(): + for found in find_external(value, key=key): + yield found + elif isinstance(tree, list): + for item in tree: + for found in find_external(item, key=key): + yield found + elif isinstance(tree, str): + text = tree.encode("utf-8") + for find in (match_fileurl, match_filename): + found = find(key, text) + if found: + yield found + + with open(path, "rb") as fd: + content = fd.read() + tree = parse_xml(content) + for found in find_external(tree): + if found.find(config.gs_data_dir) != 0: + copy_external_resource(found) + + def is_xml_file(filename, regexp=re.compile(".*.xml$")): + return regexp.match(filename) is not None + + for directory in ("workspaces", "styles"): + source = os.path.join(config.gs_data_dir, directory) + logger.info(f"Dumping external dir {source}...") + for root, dirs, files in os.walk(source): + for filename in filter(is_xml_file, files): + path = os.path.join(root, filename) + dump_external_resources_from_xml(path) diff --git a/geonode/br/management/commands/create_tile_layers.py b/geonode/br/management/commands/create_tile_layers.py new file mode 100644 index 0000000..57b81d8 --- /dev/null +++ b/geonode/br/management/commands/create_tile_layers.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import requests +from requests.auth import HTTPBasicAuth + +from django.core.management.base import BaseCommand +from django.conf import settings + +from geonode.layers.models import Dataset + + +logger = logging.getLogger(__name__) + + +REQ_TEMPLATE = """ + + true + true + {} + + 2 + 1 + + + application/json;type=utfgrid + image/gif + image/jpeg + image/png + image/png8 + image/vnd.jpeg-png + image/vnd.jpeg-png8 + + + + EPSG:3857 + + + EPSG:3857x2 + + + EPSG:4326 + + + EPSG:4326x2 + + + EPSG:900913 + + + 0 + 0 + true + 0 + +""" + + +class Command(BaseCommand): + help = "Create missing TileLayers in GWC" + + def add_arguments(self, parser): + parser.add_argument( + '-f', + '--force', + dest="force", + action='store_true', + help="Force tile layer re-creation also if it already exists in GWC") + + parser.add_argument( + '-l', + '--layer', + dest="layers", + action='append', + help="Only process specified layers ") + + parser.add_argument( + '-d', + '--dry-run', + dest="dry-run", + action='store_true', + help="Do not actually perform any change on GWC") + + def handle(self, **options): + force = options.get('force') + requested_layers = options.get('layers') + dry_run = options.get('dry-run') + + logger.debug(f"FORCE is {force}") + logger.debug(f"DRY-RUN is {dry_run}") + logger.debug(f"LAYERS is {requested_layers}") + + try: + baseurl = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] + """ + curl -v -u admin:geoserver -XGET \ + "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" + """ + layers = Dataset.objects.all() + tot = len(layers) + logger.info(f"Total layers in GeoNode: {tot}") + i = 0 + cnt_old = 0 + cnt_new = 0 + cnt_bad = 0 + cnt_skip = 0 + cnt_force = 0 + for layer in layers: + i += 1 + logger.info(f"- {i}/{tot} Processing layer: {layer.typename}") + + if requested_layers and layer.typename not in requested_layers: + logger.info(" - Layer filtered out by args") + cnt_skip += 1 + continue + + r = requests.get(f"{baseurl}gwc/rest/layers/{layer.typename}.xml", auth=HTTPBasicAuth(user, passwd)) + + if r.status_code == 200: + if force: + logger.info(" - Forcing layer configuration in GWC") + cnt_force += 1 + else: + logger.info(" - Layer already configured in GWC") + cnt_old += 1 + continue + try: + data = REQ_TEMPLATE.format(layer.name) + url = f"{baseurl}gwc/rest/layers/{layer.typename}.xml" + logger.info(" - Configuring...") + + if not dry_run: + response = requests.put( + url, data=data, headers={"Content-Type": "text/xml"}, auth=HTTPBasicAuth(user, passwd) + ) + + if dry_run or response.status_code == 200: + logger.info(f" - Done {layer.name}") + cnt_new += 1 + else: + logger.warning(f"Layer {layer.typename} couldn't be configured: code {response.status_code}") + cnt_bad += 1 + + except Exception as e: + raise e + except Exception as e: + raise e + + logger.info("Work completed") + logger.info(f"- TileLayers configured: {cnt_new}" + (f" (forced {cnt_force})" if cnt_force else "")) + logger.info(f"- TileLayers in error : {cnt_bad}") + logger.info(f"- TileLayers untouched : {cnt_old}") + logger.info(f"- TileLayers skipped : {cnt_skip}") diff --git a/geonode/br/management/commands/restore.py b/geonode/br/management/commands/restore.py new file mode 100644 index 0000000..012122a --- /dev/null +++ b/geonode/br/management/commands/restore.py @@ -0,0 +1,751 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import json +import time +import uuid +import shutil +import logging +import zipfile +import requests +import tempfile +import warnings +import pathlib +from typing import Union +from datetime import datetime + +from .utils import utils + +from distutils import dir_util +from requests.auth import HTTPBasicAuth +from urllib.parse import urlparse, urljoin + +from geonode.br.models import RestoredBackup +from geonode.br.tasks import restore_notification +from geonode.utils import DisableDjangoSignals, copy_tree, extract_archive, chmod_tree +from geonode.base.models import Configuration + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.management import call_command +from django.core.management.base import BaseCommand, CommandError +from django.db.utils import IntegrityError + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Restore the GeoNode application data" + + def add_arguments(self, parser): + # Named (optional) arguments + utils.option(parser) + + parser.add_argument( + "--geoserver-data-dir", + dest="gs_data_dir", + default=None, + help="Geoserver data directory") + + parser.add_argument( + "-i", + "--ignore-errors", + action="store_true", + dest="ignore_errors", + default=False, + help="Stop after any errors are encountered.", + ) + + parser.add_argument( + "-f", + "--force", + action="store_true", + dest="force_exec", + default=False, + help="Forces the execution without asking for confirmation.", + ) + + parser.add_argument( + "--skip-geoserver", action="store_true", dest="skip_geoserver", default=False, help="Skips geoserver backup" + ) + + parser.add_argument( + "--skip-geoserver-info", + action="store_true", + dest="skip_geoserver_info", + default=True, + help="Skips geoserver Global Infos", + ) + + parser.add_argument( + "--skip-geoserver-security", + action="store_true", + dest="skip_geoserver_security", + default=True, + help="Skips geoserver Security Settings", + ) + + parser.add_argument( + "--backup-file", dest="backup_file", default=None, help="Backup archive containing GeoNode data to restore." + ) + + parser.add_argument( + "--recovery-file", + dest="recovery_file", + default=None, + help="Archive that shall be used to restore the original content of GeoNode should the restore fail.", + ) + + parser.add_argument( + "--backup-files-dir", + dest="backup_files_dir", + default=None, + help="Directory containing GeoNode backups. Restoration procedure will pick " + "the newest created/modified backup which wasn't yet restored, and was " + "created after creation date of the currently used backup (the newest " + "if no backup was yet restored on the GeoNode instance).", + ) + + parser.add_argument( + "-l", + "--with-logs", + action="store_true", + dest="with_logs", + default=False, + help="Compares the backup file with restoration logs, and applies it only, if it hasn't been already restored", # noqa + ) + + parser.add_argument( + "-n", + "--notify", + action="store_true", + dest="notify", + default=False, + help="Sends an email notification to the superusers on procedure error or finish.", + ) + + parser.add_argument( + "--skip-read-only", + action="store_true", + dest="skip_read_only", + default=False, + help="Skips activation of the Read Only mode in restore procedure execution.", + ) + + parser.add_argument( + "--soft-reset", + action="store_true", + dest="soft_reset", + default=False, + help="If True, preserve geoserver resources and tables", + ) + + parser.add_argument( + "--skip-logger-setup", + action="store_false", + dest="setup_logger", + help='Skips setup of the "geonode.br" logger, "br" handler and "br" format if not present in settings', + ) + + def handle(self, **options): + skip_read_only = options.get("skip_read_only") + config = Configuration.load() + + # activate read only mode and store its original config value + if not skip_read_only: + original_read_only_value = config.read_only + config.read_only = True + config.save() + + if options.get("setup_logger"): + utils.setup_logger() + + try: + self.execute_restore(**options) + except Exception: + raise + finally: + # restore read only mode's original value + if not skip_read_only: + config.read_only = original_read_only_value + config.save() + + def execute_restore(self, **options): + self.validate_backup_file_options(**options) + ignore_errors = options.get("ignore_errors") + force_exec = options.get("force_exec") + skip_geoserver = options.get("skip_geoserver") + skip_geoserver_info = options.get("skip_geoserver_info") + skip_geoserver_security = options.get("skip_geoserver_security") + backup_file = options.get("backup_file") + recovery_file = options.get("recovery_file") + backup_files_dir = options.get("backup_files_dir") + with_logs = options.get("with_logs") + notify = options.get("notify") + soft_reset = options.get("soft_reset") + + # choose backup_file from backup_files_dir, if --backup-files-dir was provided + if backup_files_dir: + logger.info("*** Looking for backup file...") + backup_file = self.parse_backup_files_dir(backup_files_dir) + else: + backup_files_dir = os.path.dirname(backup_file) + + # calculate and validate backup archive hash + logger.info("*** Validating backup file...") + backup_md5 = self.validate_backup_file_hash(backup_file) + + # check if the original backup file ini setting is available or not + backup_ini = self.check_backup_ini_settings(backup_file) + if backup_ini: + options["config"] = backup_ini + config = utils.Config(options) + + # check if the backup has already been restored + if with_logs: + if RestoredBackup.objects.filter(archive_md5=backup_md5): + raise RuntimeError( + "Backup archive has already been restored. If you want to restore " + 'this backup anyway, run the script without "-l" argument.' + ) + + # get a list of instance administrators' emails + admin_emails = [] + + if notify: + admins = get_user_model().objects.filter(is_superuser=True) + for user in admins: + if user.email: + admin_emails.append(user.email) + + if not force_exec: + print("Before proceeding with the Restore, please ensure that:") + print(" 1. The backend (DB or whatever) is accessible and you have rights") + print(" 2. The GeoServer is up and running and reachable from this machine") + + message = "WARNING: The restore will overwrite ALL GeoNode data. You want to proceed?" + if force_exec or utils.confirm(prompt=message, resp=False): + # Create Target Folder + # restore_folder must be located in the directory Geoserver has access to (and it should + # not be Geoserver data dir) + # for dockerized project-template GeoNode projects, it should be located in /backup-restore, + # otherwise default tmp directory is chosen + temp_dir_path = backup_files_dir if os.path.exists(backup_files_dir) else None + + restore_folder = os.path.join( + temp_dir_path, f"unzip_{pathlib.Path(backup_file).stem}_{str(uuid.uuid4())[:4]}" + ) + try: + os.makedirs(restore_folder, exist_ok=True) + except Exception as e: + raise e + try: + # Extract ZIP Archive to Target Folder + logger.info("*** Unzipping backup file...") + target_folder = extract_archive(backup_file, restore_folder) + + # Write Checks + media_root = settings.MEDIA_ROOT + media_folder = os.path.join(target_folder, utils.MEDIA_ROOT) + assets_root = settings.ASSETS_ROOT + assets_folder = os.path.join(target_folder, utils.ASSETS_ROOT) + try: + logger.info("*** Performing some checks...") + logger.info(f"[Sanity Check] Full Write Access to restore folder: '{restore_folder}' ...") + chmod_tree(restore_folder) + logger.info(f"[Sanity Check] Full Write Access to media root: '{media_root}' ...") + chmod_tree(media_root) + logger.info(f"[Sanity Check] Full Write Access to assets root: '{assets_root}' ...") + chmod_tree(assets_root) + except Exception as e: + if notify: + restore_notification.apply_async( + args=(admin_emails, backup_file, backup_md5, str(e)), expiration=30 + ) + + logger.error( + "Sanity Checks on Folder failed. " + "Please make sure that the current user has full WRITE access to the above folders " + "(and sub-folders or files).", + exc_info=e, + ) # noqa + raise Exception(f"Some folders need write access: {str(e)}") + + if not skip_geoserver: + try: + logger.info(f"[Sanity Check] Full Write Access to target folder: '{target_folder}' ...") + chmod_tree(target_folder) + self.restore_geoserver_backup( + config, + settings, + target_folder, + skip_geoserver_info, + skip_geoserver_security, + ignore_errors, + soft_reset, + ) + self.prepare_geoserver_gwc_config(config, settings) + self.restore_geoserver_raster_data(config, settings, target_folder) + self.restore_geoserver_vector_data(config, settings, target_folder, soft_reset) + self.restore_geoserver_externals(config, settings, target_folder) + logger.info("*** Recreate GWC tile layers") + except Exception as e: + logger.warning(f"*** GeoServer Restore failed: {e}", exc_info=e) + if recovery_file: + logger.warning("*** Trying to restore from recovery file...") + with tempfile.TemporaryDirectory(dir=temp_dir_path) as restore_folder: + recovery_folder = extract_archive(recovery_file, restore_folder) + self.restore_geoserver_backup( + config, + settings, + recovery_folder, + skip_geoserver_info, + skip_geoserver_security, + ignore_errors, + soft_reset, + ) + self.restore_geoserver_raster_data(config, settings, recovery_folder) + self.restore_geoserver_vector_data(config, settings, recovery_folder, soft_reset) + self.restore_geoserver_externals(config, settings, recovery_folder) + if notify: + restore_notification.apply_async( + args=(admin_emails, backup_file, backup_md5, str(e)), expiration=30 + ) + raise Exception(f"GeoServer restore failed: {e}") + else: + logger.info("*** Skipping geoserver backup restore") + + # Prepare Target DB + try: + logger.info("*** Align the database schema") + # call_command('makemigrations', interactive=False) + call_command("migrate", interactive=False) + except Exception as e: + logger.warning(f"Error while aligning the db: {e}", exc_info=e) + + try: + # Deactivate GeoNode Signals + with DisableDjangoSignals(): + # Flush DB + try: + db_name = settings.DATABASES["default"]["NAME"] + db_user = settings.DATABASES["default"]["USER"] + db_port = settings.DATABASES["default"]["PORT"] + db_host = settings.DATABASES["default"]["HOST"] + db_passwd = settings.DATABASES["default"]["PASSWORD"] + + utils.truncate_tables(db_name, db_user, db_port, db_host, db_passwd) + except Exception: + logger.info("Error while truncating tables, trying external task") + + try: + call_command("flush", interactive=False) + except Exception as e: + logger.warning("Could not cleanup GeoNode tables", exc_info=e) + raise Exception("Could not cleanup GeoNode tables") + + # Restore Fixtures + err_cnt = 0 + + logger.info("*** Restoring GeoNode fixtures...") + + fixtures_folder = os.path.join(target_folder, "fixtures") + if not os.path.exists(fixtures_folder): + # fixtures folder was introduced on 2024-02; make the restore command lenient about + # dumps created without such a folder (this behaviour may be removed in a short while) + fixtures_folder = target_folder + + for app_name, dump_name in zip(config.app_names, config.dump_names): + fixture_file = os.path.join(fixtures_folder, f"{dump_name}.json") + + logger.info(f" - restoring '{fixture_file}'") + try: + call_command("loaddata", fixture_file, app_label=app_name) + except IntegrityError as e: + logger.warning( + f"The fixture '{dump_name}' failed the integrity check. " + "Import will be aborted after all fixtures have been checked", + exc_info=e, + ) # noqa + err_cnt += 1 + except Exception as e: + logger.warning(f"No valid fixture data found for '{dump_name}'", exc_info=e) + # helpers.load_fixture(app_name, fixture_file) + raise e + + if err_cnt: + raise IntegrityError(f"{err_cnt} fixtures could not be loaded") + + # Restore Media Root + logger.info("*** Restore media root...") + self.restore_folder(config, media_root, media_folder) + logger.info("*** Restore assets root...") + self.restore_folder(config, assets_root, assets_folder) + + # TODO improve this part, by saving the original asset_root path in a variable, then replace with the new one + + # store backup info + restored_backup = RestoredBackup( + name=backup_file.rsplit("/", 1)[-1], + archive_md5=backup_md5, + creation_date=datetime.fromtimestamp(os.path.getmtime(backup_file)), + ) + restored_backup.save() + + except Exception as e: + # exception during geonode db restore (gs has already been restored) + if notify: + restore_notification.apply_async( + args=(admin_emails, backup_file, backup_md5, str(e)), expiration=30 + ) + raise Exception(f"GeoNode restore failed: {e}") + + # call_command('makemigrations', interactive=False) + logger.info("*** Synch db with fake migrations...") + call_command("migrate", interactive=False, fake=True) + + logger.info("*** Sync layers with GeoServer...") + call_command("sync_geonode_datasets", updatepermissions=True, ignore_errors=True) + + if notify: + restore_notification.apply_async(args=(admin_emails, backup_file, backup_md5), expiration=30) + + logger.info( + "HINT: If you migrated from another site, do not forget to run the command 'migrate_baseurl' to fix Links" + ) # noqa + logger.info( + " e.g.: DJANGO_SETTINGS_MODULE=my_geonode.settings python manage.py migrate_baseurl " + "--source-address=my-host-dev.geonode.org --target-address=my-host-prod.geonode.org" + ) + logger.info("Restore finished.") + finally: + logger.info("*** Final filesystem cleanup ...") + shutil.rmtree(restore_folder) + + def restore_folder(self, config, root, folder): + if config.gs_data_dt_filter[0] is None: + shutil.rmtree(root, ignore_errors=True) + + if not os.path.exists(root): + os.makedirs(root, exist_ok=True) + + copy_tree(folder, root) + chmod_tree(root) + logger.info(f"Files restored into '{root}'.") + + def validate_backup_file_options(self, **options) -> None: + """ + Method validating --backup-file and --backup-files-dir options + + :param options: self.handle() method options + :raises: Django CommandError, if options violate requirements + :return: None + """ + + backup_file = options.get("backup_file") + backup_files_dir = options.get("backup_files_dir") + + if not any([backup_file, backup_files_dir]): + raise CommandError("Mandatory option (--backup-file|--backup-dir|--backup-files-dir)") + + if all([backup_file, backup_files_dir]): + raise CommandError("Exclusive option (--backup-file|--backup-dir|--backup-files-dir)") + + if backup_file: + if not os.path.isfile(backup_file) or not zipfile.is_zipfile(backup_file): + raise CommandError("Provided '--backup-file' is not a .zip file") + + if backup_files_dir and not os.path.isdir(backup_files_dir): + raise CommandError("Provided '--backup-files-dir' is not a directory") + + def parse_backup_files_dir(self, backup_files_dir: str) -> Union[str, None]: + """ + Method picking the Backup Archive to be restored from the Backup Files Directory. + Only archives created/modified AFTER the last restored dumps are considered. + + :param backup_files_dir: path to the directory containing backup files + :return: backup file path, if a proper backup archive was found, and None otherwise + """ + # get the latest modified backup file available in backup directory + backup_file = None + + for file_name in os.listdir(backup_files_dir): + file = os.path.join(backup_files_dir, file_name) + if zipfile.is_zipfile(file): + backup_file = ( + file + if backup_file is None or os.path.getmtime(file) > os.path.getmtime(backup_file) + else backup_file + ) # noqa + + if backup_file is None: + warnings.warn( + "Nothing to do. No backup archive found in provided '--backup-file-dir' directory", RuntimeWarning + ) + return + + # get the latest restored backup file + try: + last_restored_backup = RestoredBackup.objects.latest("restoration_date") + except RestoredBackup.DoesNotExist: + # existing if statement - backup_file will be restored, as no backup is currently loaded + pass + else: + # check if the latest modified backup file is younger than the last restored + if last_restored_backup.creation_date.timestamp() > os.path.getmtime(backup_file): + warnings.warn( + f"Nothing to do. The newest backup file from --backup-files-dir: '{backup_file}' " + "is older than the last restored backup.", + RuntimeWarning, + ) + return + + return backup_file + + def validate_backup_file_hash(self, backup_file: str) -> str: + """ + Method calculating backup file's hash and validating it, if the proper *.md5 file + exists in the backup_file directory + + :param backup_file: path to the backup_file + :return: backup_file hash + """ + # calculate backup file's md5 hash + backup_hash = utils.md5_file_hash(backup_file) + + # check md5 hash for backup archive, if the md5 file is in place + archive_md5_file = f"{backup_file.rsplit('.', 1)[0]}.md5" + + if os.path.exists(archive_md5_file): + with open(archive_md5_file) as md5_file: + original_backup_md5 = md5_file.readline().strip().split(" ")[0] + + if original_backup_md5 != backup_hash: + raise RuntimeError( + "Backup archive integrity failure. MD5 hash of the archive " + f"is different from the one provided in {archive_md5_file}" + ) + else: + warnings.warn( + "Backup archive's MD5 file does not exist under expected path. Skipping integrity check.", + RuntimeWarning, + ) + # sleep for the proper console writing order + time.sleep(0.1) + + return backup_hash + + def check_backup_ini_settings(self, backup_file: str) -> str: + """ + Method checking backup file's original settings availability + + :param backup_file: path to the backup_file + :return: backup_ini_file_path original settings used by the backup file + """ + # check if the ini file is in place + backup_ini_file_path = f"{backup_file.rsplit('.', 1)[0]}.ini" + + if os.path.exists(backup_ini_file_path): + return backup_ini_file_path + + return None + + def restore_geoserver_backup( + self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset + ): + """Restore GeoServer Catalog""" + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] + geoserver_bk_file = os.path.join(target_folder, "geoserver_catalog.zip") + + logger.info(f"*** Restoring GeoServer catalog [{url}] from '{geoserver_bk_file}'") + + if not os.path.exists(geoserver_bk_file) or not os.access(geoserver_bk_file, os.R_OK): + raise Exception(f'ERROR: geoserver restore: file "{geoserver_bk_file}" not found.') + + def bstr(x): + return "true" if x else "false" + + # Best Effort Restore: 'options': {'option': ['BK_BEST_EFFORT=true']} + _options = [ + f"BK_PURGE_RESOURCES={bstr(not soft_reset)}", + "BK_CLEANUP_TEMP=true", + f"BK_SKIP_SETTINGS={bstr(skip_geoserver_info)}", + f"BK_SKIP_SECURITY={bstr(skip_geoserver_security)}", + "BK_BEST_EFFORT=true", + f"exclude.file.path={config.gs_exclude_file_path}", + ] + data = {"restore": {"archiveFile": geoserver_bk_file, "options": {"option": _options}}} + headers = {"Accept": "application/json", "Content-type": "application/json"} + r = requests.post( + f"{url}rest/br/restore/", data=json.dumps(data), headers=headers, auth=HTTPBasicAuth(user, passwd) + ) + error_backup = "Could not successfully restore GeoServer catalog [{}rest/br/restore/]: {} - {}" + + if r.status_code in (200, 201, 406): + try: + r = requests.get( + f"{url}rest/br/restore.json", headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10 + ) + + if r.status_code == 200: + gs_backup = r.json() + _url = urlparse(gs_backup["restores"]["restore"][len(gs_backup["restores"]["restore"]) - 1]["href"]) + _url = f"{urljoin(url, _url.path)}?{_url.query}" + r = requests.get(_url, headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) + + if r.status_code == 200: + gs_backup = r.json() + + if r.status_code != 200: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + except ValueError: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + gs_bk_exec_id = gs_backup["restore"]["execution"]["id"] + r = requests.get( + f"{url}rest/br/restore/{gs_bk_exec_id}.json", + headers=headers, + auth=HTTPBasicAuth(user, passwd), + timeout=10, + ) + + if r.status_code == 200: + gs_bk_exec_status = gs_backup["restore"]["execution"]["status"] + gs_bk_exec_progress = gs_backup["restore"]["execution"]["progress"] + gs_bk_exec_progress_updated = "0/0" + while gs_bk_exec_status != "COMPLETED" and gs_bk_exec_status != "FAILED": + if gs_bk_exec_progress != gs_bk_exec_progress_updated: + gs_bk_exec_progress_updated = gs_bk_exec_progress + r = requests.get( + f"{url}rest/br/restore/{gs_bk_exec_id}.json", + headers=headers, + auth=HTTPBasicAuth(user, passwd), + timeout=10, + ) + + if r.status_code == 200: + try: + gs_backup = r.json() + except ValueError: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + gs_bk_exec_status = gs_backup["restore"]["execution"]["status"] + gs_bk_exec_progress = gs_backup["restore"]["execution"]["progress"] + logger.info(f"Async restore status: {gs_bk_exec_status} - {gs_bk_exec_progress}") + time.sleep(3) + else: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + if gs_bk_exec_status != "COMPLETED": + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + else: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + else: + raise ValueError(error_backup.format(url, r.status_code, r.text)) + + def prepare_geoserver_gwc_config(self, config, settings): + if config.gs_data_dir: + logger.info("*** Cleanup old GWC config...") + # Cleanup '$config.gs_data_dir/gwc-layers' + gwc_layers_root = os.path.join(config.gs_data_dir, "gwc-layers") + if not os.path.isabs(gwc_layers_root): + gwc_layers_root = os.path.join(settings.PROJECT_ROOT, "..", gwc_layers_root) + try: + logger.info(f"Cleaning out old GeoServer GWC layers config: {gwc_layers_root}") + shutil.rmtree(gwc_layers_root) + except Exception as e: + logger.info(f"Error while cleaning old GeoServer GWC layers config: {e}") + if not os.path.exists(gwc_layers_root): + logger.info(f"Recreating GWC layers dir: {gwc_layers_root}") + os.makedirs(gwc_layers_root, exist_ok=True) + + def restore_geoserver_raster_data(self, config, settings, target_folder): + if config.gs_data_dir and config.gs_dump_raster_data: + logger.info("*** Restore raster data") + + for dest_folder, source_root in ( + ( + os.path.join(config.gs_data_dir, "geonode"), + os.path.join(target_folder, "gs_data_dir", "geonode"), + ), # Dump '$config.gs_data_dir/geonode' + ( + os.path.join(config.gs_data_dir, "data", "geonode"), + os.path.join(target_folder, "gs_data_dir", "data", "geonode"), + ), # Dump '$config.gs_data_dir/data/geonode' + ): + if os.path.exists(source_root): + logger.info(f"Restoring raster data to '{dest_folder}'...") + if not os.path.isabs(dest_folder): + dest_folder = os.path.join(settings.PROJECT_ROOT, "..", dest_folder) + + if not os.path.exists(dest_folder): + os.makedirs(dest_folder, exist_ok=True) + + logger.info(f"Copying data from '{source_root}' to '{dest_folder}'...") + copy_tree(source_root, dest_folder) + logger.info(f"Restored raster data to '{dest_folder}'") + else: + logger.info(f"Skipping raster data directory '{source_root}' because it does not exist") + + def restore_geoserver_vector_data(self, config, settings, target_folder, soft_reset): + """Restore Vectorial Data from DB""" + if config.gs_dump_vector_data: + logger.info("*** Restore vector data") + + gs_data_folder = os.path.join(target_folder, "gs_data_dir", "geonode") + if not os.path.exists(gs_data_folder): + logger.info(f'Skipping vector data restore: directory "{gs_data_folder}" not found') + return + logger.info(f'Restoring vector data from "{gs_data_folder}" not found') + + datastore_name = settings.OGC_SERVER["default"]["DATASTORE"] + datastore = settings.DATABASES[datastore_name] + if datastore_name: + ogc_db_name = datastore["NAME"] + ogc_db_user = datastore["USER"] + ogc_db_passwd = datastore["PASSWORD"] + ogc_db_host = datastore["HOST"] + ogc_db_port = datastore["PORT"] + + if not soft_reset: + utils.remove_existing_tables(ogc_db_name, ogc_db_user, ogc_db_port, ogc_db_host, ogc_db_passwd) + + utils.restore_db( + config, + ogc_db_name, + ogc_db_user, + ogc_db_port, + ogc_db_host, + ogc_db_passwd, + gs_data_folder, + soft_reset, + ) + + def restore_geoserver_externals(self, config, settings, target_folder): + """Restore external references from XML files""" + logger.info("*** Restoring GeoServer external resources...") + external_folder = os.path.join(target_folder, utils.EXTERNAL_ROOT) + if os.path.exists(external_folder): + dir_util.copy_tree(external_folder, "/") diff --git a/geonode/br/management/commands/settings_docker_sample.ini b/geonode/br/management/commands/settings_docker_sample.ini new file mode 100644 index 0000000..f5401a9 --- /dev/null +++ b/geonode/br/management/commands/settings_docker_sample.ini @@ -0,0 +1,18 @@ +[database] +pgdump = pg_dump +pgrestore = pg_restore +psql = psql + +[geoserver] +datadir = /geoserver_data/data +# datadir_exclude_file_path = {comma separated list of paths to exclude from geoserver catalog} e.g.: /data,/data/geonode,/geonode +dumpvectordata = yes +dumprasterdata = yes +# data_dt_filter = {cmp_operator} {ISO8601} e.g. > 20019-04-05T24:00 +# data_layername_filter = {comma separated list of layernames, optionally with glob syntax} e.g.: tuscany_*,italy +# data_layername_exclude_filter = {comma separated list of layernames, optionally with glob syntax} e.g.: tuscany_*,italy + +[fixtures] +apps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client +dumps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client + diff --git a/geonode/br/management/commands/settings_sample.ini b/geonode/br/management/commands/settings_sample.ini new file mode 100644 index 0000000..112b661 --- /dev/null +++ b/geonode/br/management/commands/settings_sample.ini @@ -0,0 +1,17 @@ +[database] +pgdump = pg_dump +pgrestore = pg_restore +psql = psql + +[geoserver] +datadir = geoserver/data +# datadir_exclude_file_path = {comma separated list of paths to exclude from geoserver catalog} e.g.: /data,/data/geonode,/geonode +dumpvectordata = yes +dumprasterdata = yes +# data_dt_filter = {cmp_operator} {ISO8601} e.g. > 2019-04-05T24:00 +# data_layername_filter = {comma separated list of layernames, optionally with glob syntax} e.g.: tuscany_*, +# data_layername_exclude_filter = {comma separated list of layernames, optionally with glob syntax} e.g.: tuscany_*,italy + +[fixtures] +apps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client +dumps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client diff --git a/geonode/br/management/commands/utils/__init__.py b/geonode/br/management/commands/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/br/management/commands/utils/utils.py b/geonode/br/management/commands/utils/utils.py new file mode 100644 index 0000000..4b48399 --- /dev/null +++ b/geonode/br/management/commands/utils/utils.py @@ -0,0 +1,482 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import re +import sys +import shutil +import hashlib +from logging import Formatter, StreamHandler + +import psycopg2 +import traceback +import dateutil.parser +import logging +import subprocess + +from configparser import ConfigParser + +from django.conf import settings +from django.core.management.base import CommandError + + +MEDIA_ROOT = "uploaded" +STATIC_ROOT = "static_root" +STATICFILES_DIRS = "static_dirs" +TEMPLATE_DIRS = "template_dirs" +LOCALE_PATHS = "locale_dirs" +EXTERNAL_ROOT = "external" +ASSETS_ROOT = "assets" + + +logger = logging.getLogger(__name__) + + +def option(parser): + # Named (optional) arguments + parser.add_argument("-c", "--config", help="Use custom settings.ini configuration file") + + +def geoserver_option_list(parser): + # Named (optional) arguments + parser.add_argument("--geoserver-data-dir", dest="gs_data_dir", default=None, help="Geoserver data directory") + + parser.add_argument( + "--dump-geoserver-vector-data", + dest="dump_gs_vector_data", + action="store_true", + default=None, + help="Dump geoserver vector data", + ) + + parser.add_argument( + "--no-geoserver-vector-data", + dest="dump_gs_vector_data", + action="store_false", + default=None, + help="Don't dump geoserver vector data", + ) + + parser.add_argument( + "--dump-geoserver-raster-data", + dest="dump_gs_raster_data", + action="store_true", + default=None, + help="Dump geoserver raster data", + ) + + parser.add_argument( + "--no-geoserver-raster-data", + dest="dump_gs_raster_data", + action="store_false", + default=None, + help="Don't dump geoserver raster data", + ) + + +class Config: + def __init__(self, options: dict): + def apply_options_override(options): + def get_option(key, fallback): + o = options.get(key) + return o if o is not None else fallback + + self.gs_data_dir = get_option("gs_data_dir", self.gs_data_dir) + self.gs_dump_vector_data = get_option("dump_gs_vector_data", self.gs_dump_vector_data) + self.gs_dump_raster_data = get_option("dump_gs_raster_data", self.gs_dump_raster_data) + + # store back overrides as current config (needed for saving it into the backup zip) + self.config_parser["geoserver"]["datadir"] = self.gs_data_dir + self.config_parser["geoserver"]["dumpvectordata"] = str(self.gs_dump_vector_data) + self.config_parser["geoserver"]["dumprasterdata"] = str(self.gs_dump_raster_data) + + def load_settings(config): + self.pg_dump_cmd = config.get("database", "pgdump") + self.pg_restore_cmd = config.get("database", "pgrestore") + self.psql_cmd = config.get("database", "psql", fallback="psql") + + self.gs_data_dir = config.get("geoserver", "datadir") + + self.gs_exclude_file_path = ( + ";".join(config.get("geoserver", "datadir_exclude_file_path").split(",")) + if config.has_option("geoserver", "datadir_exclude_file_path") + else "" + ) + + self.gs_dump_vector_data = config.getboolean("geoserver", "dumpvectordata") + self.gs_dump_raster_data = config.getboolean("geoserver", "dumprasterdata") + + self.gs_data_dt_filter = ( + config.get("geoserver", "data_dt_filter").split(" ") + if config.has_option("geoserver", "data_dt_filter") + else (None, None) + ) + + self.gs_data_datasetname_filter = ( + config.get("geoserver", "data_datasetname_filter").split(",") + if config.has_option("geoserver", "data_datasetname_filter") + else "" + ) + + self.gs_data_datasetname_exclude_filter = ( + config.get("geoserver", "data_datasetname_exclude_filter").split(",") + if config.has_option("geoserver", "data_datasetname_exclude_filter") + else "" + ) + + self.app_names = config.get("fixtures", "apps").split(",") + self.dump_names = config.get("fixtures", "dumps").split(",") + + # Start init code + settings_path = options.get("config") + if not settings_path: + raise CommandError("Missing mandatory option (-c / --config)") + if not os.path.isabs(settings_path): + settings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, settings_path) + if not os.path.exists(settings_path): + raise CommandError(f"Provided '-c' / '--config' file does not exist: {settings_path}") + + self.config_parser = ConfigParser() + self.config_parser.read(settings_path) + + # set config from file + load_settings(self.config_parser) + # override config from command line + apply_options_override(options) + + +sys.path.append(os.path.join(os.path.dirname(__file__), "lib")) + + +def get_db_conn(db_name, db_user, db_port, db_host, db_passwd): + """Get db conn (GeoNode)""" + db_host = db_host if db_host is not None else "localhost" + db_port = db_port if db_port is not None else 5432 + conn = psycopg2.connect( + f"dbname='{db_name}' user='{db_user}' port='{db_port}' host='{db_host}' password='{db_passwd}'" + ) + return conn + + +def get_tables(db_user, db_passwd, db_name, db_host="localhost", db_port=5432): + select = f"SELECT tablename FROM pg_tables WHERE tableowner = '{db_user}' and schemaname = 'public'" + logger.info(f"Retrieving table list from DB {db_name}@{db_host}: {select}") + + try: + conn = get_db_conn(db_name, db_user, db_port, db_host, db_passwd) + curs = conn.cursor() + curs.execute(select) + pg_tables = [table[0] for table in curs.fetchall()] + conn.commit() + return pg_tables + + except Exception as e: + traceback.print_exc() + raise e + finally: + curs.close() + conn.close() + + +def truncate_tables(db_name, db_user, db_port, db_host, db_passwd): + """HARD Truncate all DB Tables""" + db_host = db_host if db_host is not None else "localhost" + db_port = db_port if db_port is not None else 5432 + + logger.info(f"Truncating the tables in DB {db_name} @{db_host}:{db_port} for user {db_user}") + pg_tables = get_tables(db_user, db_passwd, db_name, db_host, db_port) + logger.info(f"Tables found: {pg_tables}") + + conn = get_db_conn(db_name, db_user, db_port, db_host, db_passwd) + bad_tables = [] + + try: + for table in sorted(pg_tables): + if table == "br_restoredbackup": + continue + logger.info(f"Truncating table : {table}") + try: + curs = conn.cursor() + curs.execute(f"TRUNCATE {table} CASCADE;") + conn.commit() + except Exception as e: + logger.warning(f"Could not truncate table {table}: {e}", exc_info=e) + bad_tables.append(table) + conn.rollback() + if bad_tables: + raise Exception(f"Could not truncate tables {bad_tables}") + + finally: + curs.close() + conn.close() + + +def dump_db(config, db_name, db_user, db_port, db_host, db_passwd, target_folder): + """Dump Full DB into target folder""" + db_host = db_host if db_host is not None else "localhost" + db_port = db_port if db_port is not None else 5432 + + logger.info("Dumping data tables") + pg_tables = get_tables(db_user, db_passwd, db_name, db_host, db_port) + logger.info(f"Tables found: {pg_tables}") + + include_filter = config.gs_data_datasetname_filter + exclude_filter = config.gs_data_datasetname_exclude_filter + + if include_filter: + filtered_tables = [] + for pat in include_filter: + filtered_tables += glob_filter(pg_tables, pat) + pg_tables = filtered_tables + logger.info(f"Tables found after INCLUDE filtering: {pg_tables}") + + elif exclude_filter: + for pat in exclude_filter: + names = glob_filter(pg_tables, pat) + for exclude_table in names: + pg_tables.remove(exclude_table) + logger.info(f"Tables found after EXCLUDE filtering: {pg_tables}") + + logger.debug(f"Cleaning up destination folder {target_folder}...") + empty_folder(target_folder) + for table in sorted(pg_tables): + logger.info(f" - Dumping data table: {db_name}:{table}") + command = ( + f"{config.pg_dump_cmd} " + f" -h {db_host} -p {str(db_port)} -U {db_user} -d {db_name} " + f" -b " + f" -t '\"{str(table)}\"' " + f" -f {os.path.join(target_folder, f'{table}.sql ')}" + ) + ret = subprocess.call(command, shell=True, env={"PGPASSWORD": db_passwd}) + if ret != 0: + logger.error(f"DUMP FAILED FOR TABLE {table}") + + +def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_folder, preserve_tables): + """Restore Full DB into target folder""" + db_host = db_host if db_host is not None else "localhost" + db_port = db_port if db_port is not None else 5432 + + logger.info("Restoring data tables") + + dump_extensions = ["dump", "sql"] + file_names = [fn for fn in os.listdir(source_folder) if any(fn.endswith(ext) for ext in dump_extensions)] + for filename in sorted(file_names): + table_name = os.path.splitext(filename)[0] + logger.info(f" - restoring data table: {db_name}:{table_name} ") + if filename.endswith("dump"): + command = ( + f"{config.pg_restore_cmd} " + f" -h {db_host} -p {str(db_port)} -d {db_name}" + f" -U {db_user} --role={db_user} " + f' -t "{table_name}" ' + f' {"-c" if not preserve_tables else "" } ' + f" {os.path.join(source_folder, filename)} " + ) + ret = subprocess.call(command, env={"PGPASSWORD": db_passwd}, shell=True) + if ret: + logger.error(f"RESTORE FAILED FOR FILE {filename}") + + elif filename.endswith("sql"): + args = ( + f"{config.psql_cmd} " + f" -h {db_host} " + f" -p {str(db_port)} " + f" -d {db_name} " + f" -U {db_user} " + f" -f {os.path.join(source_folder, filename)} " + " -q -b " + ) + cproc = subprocess.run(args, env={"PGPASSWORD": db_passwd}, shell=True, capture_output=True, text=True) + ret = cproc.returncode + if ret: + logger.error(f"RESTORE FAILED FOR FILE {filename}") + logger.error(f'CMD:: {" ".join(args)}') + # logger.error(f'OUT:: {cproc.stdout}') + logger.error(f"ERR:: {cproc.stderr}") + + +def remove_existing_tables(db_name, db_user, db_port, db_host, db_passwd): + logger.info("Dropping existing GeoServer vector data from DB") + pg_tables = get_tables(db_user, db_passwd, db_name, db_host, db_port) + bad_tables = [] + + conn = get_db_conn(db_name, db_user, db_port, db_host, db_passwd) + + for table in pg_tables: + logger.info(f"- Drop Table: {db_name}:{table} ") + try: + curs = conn.cursor() + curs.execute(f'DROP TABLE "{table}" CASCADE') + conn.commit() + except Exception as e: + logger.warning(f"Error Dropping Table {table}: {str(e)}", exc_info=e) + bad_tables.append(table) + conn.rollback() + + if bad_tables: + logger.warning("Some tables could not be removed. This error will probably break the procedure in next steps.") + logger.warning(f"Bad tables list: {bad_tables}") + + curs.close() + conn.close() + + +def confirm(prompt=None, resp=False): + """prompts for yes or no response from the user. Returns True for yes and + False for no. + + 'resp' should be set to the default value assumed by the caller when + user simply types ENTER. + + >>> confirm(prompt='Create Directory?', resp=True) + Create Directory? [y]|n: + True + >>> confirm(prompt='Create Directory?', resp=False) + Create Directory? [n]|y: + False + >>> confirm(prompt='Create Directory?', resp=False) + Create Directory? [n]|y: y + True + """ + + if prompt is None: + prompt = "Confirm" + + if resp: + prompt = f"{prompt} [y]|n: " + else: + prompt = f"{prompt} [n]|y: " + + while True: + ans = input(prompt) + if not ans: + return resp + if ans not in {"y", "Y", "n", "N"}: + print("please enter y or n.") + continue + if ans == "y" or ans == "Y": + return True + if ans == "n" or ans == "N": + return False + + +def md5_file_hash(file_path): + """ + A method generating MD5 hash of the provided file. + + :param file_path: file's path with an extension, which will be opened for reading and generating md5 hash + :return: hex representation of md5 hash + """ + hash_md5 = hashlib.md5() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def ignore_time(cmp_operator, iso_date): + def ignoref(directory, contents): + if not cmp_operator or not iso_date: + return [] + _timestamp = dateutil.parser.isoparse(iso_date).timestamp() + if cmp_operator == "<": + return (f for f in contents if os.path.getmtime(os.path.join(directory, f)) > _timestamp) + elif cmp_operator == "<=": + return (f for f in contents if os.path.getmtime(os.path.join(directory, f)) >= _timestamp) + elif cmp_operator == "=": + return (f for f in contents if os.path.getmtime(os.path.join(directory, f)) == _timestamp) + elif cmp_operator == ">": + return (f for f in contents if os.path.getmtime(os.path.join(directory, f)) < _timestamp) + elif cmp_operator == ">=": + return (f for f in contents if os.path.getmtime(os.path.join(directory, f)) <= _timestamp) + + return ignoref + + +def glob2re(pat): + """Translate a shell PATTERN to a regular expression. + + There is no way to quote meta-characters. + """ + + i, n = 0, len(pat) + res = "" + while i < n: + c = pat[i] + i = i + 1 + if c == "*": + # res = res + '.*' + res = f"{res}[^/]*" + elif c == "?": + # res = res + '.' + res = f"{res}[^/]" + elif c == "[": + j = i + if j < n and pat[j] == "!": + j = j + 1 + if j < n and pat[j] == "]": + j = j + 1 + while j < n and pat[j] != "]": + j = j + 1 + if j >= n: + res = f"{res}\\[" + else: + stuff = pat[i:j].replace("\\", "\\\\") + i = j + 1 + if stuff[0] == "!": + stuff = f"^{stuff[1:]}" + elif stuff[0] == "^": + stuff = f"\\{stuff}" + res = f"{res}[{stuff}]" + else: + res = res + re.escape(c) + return f"{res}\\Z(?ms)" + + +def glob_filter(names, pat): + return (name for name in names if re.match(glob2re(pat), name)) + + +def empty_folder(folder): + for filename in os.listdir(folder): + file_path = os.path.join(folder, filename) + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + print(f"Failed to delete {file_path}. Reason: {e}") + + +def setup_logger(): + if "geonode.br" not in settings.LOGGING["loggers"]: + settings.LOGGING["formatters"]["br"] = {"format": "%(levelname)-7s %(asctime)s %(message)s"} + settings.LOGGING["handlers"]["br"] = {"level": "DEBUG", "class": "logging.StreamHandler", "formatter": "br"} + settings.LOGGING["loggers"]["geonode.br"] = {"handlers": ["br"], "level": "INFO", "propagate": False} + + logger = logging.getLogger("geonode.br") + + handler = StreamHandler() + handler.setFormatter(Formatter(fmt="%(levelname)-7s %(asctime)s %(message)s")) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.propagate = False diff --git a/geonode/br/migrations/0001_initial.py b/geonode/br/migrations/0001_initial.py new file mode 100644 index 0000000..e18c0b4 --- /dev/null +++ b/geonode/br/migrations/0001_initial.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.11 on 2020-03-23 14:36 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RestoredBackup", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(blank=True, max_length=400)), + ("archive_md5", models.CharField(max_length=32)), + ("restoration_date", models.DateTimeField(default=datetime.datetime.now)), + ("creation_date", models.DateTimeField()), + ], + ), + ] diff --git a/geonode/br/migrations/__init__.py b/geonode/br/migrations/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/br/migrations/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/br/models.py b/geonode/br/models.py new file mode 100644 index 0000000..c1c8f0e --- /dev/null +++ b/geonode/br/models.py @@ -0,0 +1,32 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +from datetime import datetime + +from django.db import models + +logger = logging.getLogger(__name__) + + +class RestoredBackup(models.Model): + name = models.CharField(blank=True, max_length=400) + archive_md5 = models.CharField(blank=False, null=False, max_length=32) + restoration_date = models.DateTimeField(blank=False, null=False, default=datetime.now) + creation_date = models.DateTimeField() diff --git a/geonode/br/tasks.py b/geonode/br/tasks.py new file mode 100644 index 0000000..52bbfc6 --- /dev/null +++ b/geonode/br/tasks.py @@ -0,0 +1,59 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from typing import List + +from django.conf import settings +from django.core.mail import EmailMessage + +from geonode.celery_app import app + + +@app.task( + bind=True, + name="geonode.br.tasks.restore_notification", + queue="email", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def restore_notification(self, recipients: List, backup_file: str, backup_md5: str, exception: str = None): + """ + Function sending a CC email report of the restore procedure to a provided emails. + """ + if exception: + subject = "Geonode restore procedure FAILED." + message = ( + f'Restoration of the backup file: "{backup_file}" (MD5 hash: {backup_md5}) on the ' + f"GeoNode instance: {settings.SITEURL} FAILED with an exception: {exception}" + ) + else: + subject = "Geonode restore procedure finished with SUCCESS." + message = ( + f'Restoration of the backup file: "{backup_file}" (MD5 hash: {backup_md5}) on the ' + f"GeoNode instance: {settings.SITEURL} was finished SUCCESSFULLY." + ) + + msg = EmailMessage(subject=subject, body=message, to=recipients) + msg.send() diff --git a/geonode/br/tests/__init__.py b/geonode/br/tests/__init__.py new file mode 100644 index 0000000..36ea6cf --- /dev/null +++ b/geonode/br/tests/__init__.py @@ -0,0 +1,22 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.br.tests.test_restore import * # noqa aside +from geonode.br.tests.test_restore_helpers import * # noqa aside +from geonode.br.tests.test_backup import * # noqa aside diff --git a/geonode/br/tests/factories.py b/geonode/br/tests/factories.py new file mode 100644 index 0000000..cabf772 --- /dev/null +++ b/geonode/br/tests/factories.py @@ -0,0 +1,47 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import factory +import random +import string +import hashlib +from datetime import datetime +from factory.django import DjangoModelFactory + +from geonode.br.models import RestoredBackup + + +def random_md5_hash() -> str: + """ + Method calculating MD5 hash of a random string + + :return: hex representation of md5 hash of a random string + """ + return hashlib.md5( + "".join(random.choices(string.ascii_uppercase + string.digits, k=15)).encode("utf-8") + ).hexdigest() + + +class RestoredBackupFactory(DjangoModelFactory): + class Meta: + model = RestoredBackup + + name = factory.Faker("word") + archive_md5 = factory.LazyFunction(random_md5_hash) + restoration_date = factory.LazyFunction(datetime.now) + creation_date = factory.Faker("date_time") diff --git a/geonode/br/tests/test_backup.py b/geonode/br/tests/test_backup.py new file mode 100644 index 0000000..14700e5 --- /dev/null +++ b/geonode/br/tests/test_backup.py @@ -0,0 +1,105 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +from unittest import mock +import tempfile + +from django.core.management import call_command +from django.core.management.base import CommandError + +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.base.models import Configuration + + +class BackupCommandTests(GeoNodeBaseTestSupport): + def setUp(self): + super().setUp() + # make sure Configuration exists in the database for Read Only mode tests + Configuration.load() + + # force backup interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + # mock geonode.base.models.Configuration save() method + @mock.patch("geonode.base.models.Configuration.save", return_value=None) + def test_with_read_only_mode(self, mock_configuration_save, fake_confirm): + with tempfile.TemporaryDirectory() as tmp_dir: + args = [] + kwargs = { + "backup_dir": tmp_dir, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), + } + + call_command("backup", *args, **kwargs) + + # make sure Configuration was saved twice (Read-Only set, and revert) + self.assertEqual(mock_configuration_save.call_count, 2) + + # force backup interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + # mock geonode.base.models.Configuration save() method + @mock.patch("geonode.base.models.Configuration.save", return_value=None) + def test_without_read_only_mode(self, mock_configuration_save, fake_confirm): + with tempfile.TemporaryDirectory() as tmp_dir: + args = ["--skip-read-only"] + kwargs = { + "backup_dir": tmp_dir, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), + } + + call_command("backup", *args, **kwargs) + + # make sure Configuration wasn't called at all + self.assertEqual(mock_configuration_save.call_count, 0) + + # force backup interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + # mock geonode.base.models.Configuration save() method + @mock.patch("geonode.base.models.Configuration.save", return_value=None) + def test_config_file_not_provided(self, mock_configuration_save, fake_confirm): + with tempfile.TemporaryDirectory() as tmp_dir: + args = ["--skip-read-only"] + kwargs = {"backup_dir": tmp_dir} + + with self.assertRaises(CommandError) as exc: + call_command("backup", *args, **kwargs) + + self.assertIn( + "andatory option (-c / --config)", + exc.exception.args[0], + 'Can not match message about mandatory option (-c / --config)" exception', + ) + + # force backup interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + # mock geonode.base.models.Configuration save() method + @mock.patch("geonode.base.models.Configuration.save", return_value=None) + def test_config_file_does_not_exist(self, mock_configuration_save, fake_confirm): + with tempfile.TemporaryDirectory() as tmp_dir: + args = ["--skip-read-only"] + kwargs = {"backup_dir": tmp_dir, "config": "/some/random/path"} + + with self.assertRaises(CommandError) as exc: + call_command("backup", *args, **kwargs) + + self.assertIn("file does not exist", exc.exception.args[0], '"file does not exist" exception expected.') diff --git a/geonode/br/tests/test_restore.py b/geonode/br/tests/test_restore.py new file mode 100644 index 0000000..4450e15 --- /dev/null +++ b/geonode/br/tests/test_restore.py @@ -0,0 +1,178 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +from unittest import mock +import zipfile +import tempfile + +from django.core.management import call_command +from django.core.management.base import CommandError + +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.br.tests.factories import RestoredBackupFactory +from geonode.br.management.commands.utils.utils import md5_file_hash +from geonode.base.models import Configuration + + +class RestoreCommandTests(GeoNodeBaseTestSupport): + def setUp(self): + super().setUp() + # make sure Configuration exists in the database for Read Only mode tests + Configuration.load() + + # force restore interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + def test_with_logs_success(self, fake_confirm): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + # create an entry in restoration history with the same dump's hash + RestoredBackupFactory(archive_md5="91162629d258a876ee994e9233b2ad87") + + args = ["-l"] + kwargs = { + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), + } + + call_command("restore", *args, **kwargs) + + # force restore interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + def test_with_logs_failure(self, fake_confirm): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + # create an entry in restoration history with the same dump's hash + RestoredBackupFactory(archive_md5=md5_file_hash(tmp_file.name)) + + args = ["-l"] + kwargs = { + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), + } + + with self.assertRaises(RuntimeError) as exc: + call_command("restore", *args, **kwargs) + + self.assertIn( + "Backup archive has already been restored", + exc.exception.args[0], + '"Backup archive has already been restored" exception expected.', + ) + + # force backup interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + # mock geonode.base.models.Configuration save() method + @mock.patch("geonode.base.models.Configuration.save", return_value=None) + def test_with_read_only_mode(self, mock_configuration_save, fake_confirm): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + args = [] + kwargs = { + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), + } + + call_command("restore", *args, **kwargs) + + # make sure Configuration was saved twice (Read-Only set, and revert) + self.assertEqual(mock_configuration_save.call_count, 2) + + # force backup interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + # mock geonode.base.models.Configuration save() method + @mock.patch("geonode.base.models.Configuration.save", return_value=None) + def test_without_read_only_mode(self, mock_configuration_save, fake_confirm): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + args = ["--skip-read-only"] + kwargs = { + "backup_file": tmp_file.name, + "config": os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "management/commands/settings_sample.ini" + ), + } + + call_command("restore", *args, **kwargs) + + # make sure Configuration wasn't called at all + self.assertEqual(mock_configuration_save.call_count, 0) + + # force backup interruption before starting the procedure itself + @mock.patch("geonode.br.management.commands.utils.utils.confirm", return_value=False) + # mock geonode.base.models.Configuration save() method + @mock.patch("geonode.base.models.Configuration.save", return_value=None) + def test_config_files(self, mock_configuration_save, fake_confirm): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + args = ["--skip-read-only"] + kwargs = {"backup_file": tmp_file.name} + + with self.assertRaises(CommandError) as exc: + call_command("restore", *args, **kwargs) + + self.assertIn( + "andatory option (-c / --config)", + exc.exception.args[0], + 'Can not match message about mandatory option (-c / --config)" exception', + ) + + # create the backup file with ini file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + tmp_ini_file = f"{tmp_file.name.rsplit('.', 1)[0]}.ini" + from configparser import ConfigParser + + config = ConfigParser() + config["database"] = {"pgdump": "pg_dump", "pgrestore": "pg_restore"} + + config["geoserver"] = {"datadir": "geoserver/data", "dumpvectordata": "yes", "dumprasterdata": "yes"} + + config["fixtures"] = {"apps": "fake", "dumps": "fake"} + + with open(tmp_ini_file, "w") as configfile: + config.write(configfile) + + args = ["--skip-read-only"] + kwargs = {"backup_file": tmp_file.name} + + call_command("restore", *args, **kwargs) diff --git a/geonode/br/tests/test_restore_helpers.py b/geonode/br/tests/test_restore_helpers.py new file mode 100644 index 0000000..b950dfa --- /dev/null +++ b/geonode/br/tests/test_restore_helpers.py @@ -0,0 +1,205 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import time +import zipfile +import tempfile +import datetime + +from django.core.management.base import CommandError + +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.br.tests.factories import RestoredBackupFactory +from geonode.br.management.commands.utils.utils import md5_file_hash +from geonode.br.management.commands.restore import Command as RestoreCommand + + +class RestoreCommandHelpersTests(GeoNodeBaseTestSupport): + # validate_backup_file_hash + + # validate_backup_file_options() method test + def test_mandatory_option_failure(self): + options = {} + + with self.assertRaises(CommandError) as exc: + RestoreCommand().validate_backup_file_options(**options) + + self.assertIn("Mandatory option", exc.exception.args[0], '"Mandatory option" failure exception expected.') + + # validate_backup_file_options() method test + def test_exclusive_option_failure(self): + options = {"backup_file": "/some/random/path/1.zip", "backup_files_dir": "/some/random/path"} + + with self.assertRaises(CommandError) as exc: + RestoreCommand().validate_backup_file_options(**options) + + self.assertIn("Exclusive option", exc.exception.args[0], '"Exclusive option" failure exception expected.') + + # validate_backup_file_options() method test + def test_mandatory_option_backup_file_success(self): + with tempfile.NamedTemporaryFile() as tmp: + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + options = {"backup_file": tmp.name} + RestoreCommand().validate_backup_file_options(**options) + + # validate_backup_file_options() method test + def test_mandatory_option_backup_file_failure(self): + with tempfile.NamedTemporaryFile() as tmp: + options = {"backup_file": tmp.name} + + with self.assertRaises(CommandError) as exc: + RestoreCommand().validate_backup_file_options(**options) + + self.assertIn("not a .zip file", exc.exception.args[0], '"not a .zip file" failure exception expected.') + + # validate_backup_file_options() method test + def test_mandatory_option_backup_files_dir_success(self): + with tempfile.TemporaryDirectory() as tmp: + options = {"backup_files_dir": tmp} + RestoreCommand().validate_backup_file_options(**options) + + # validate_backup_file_options() method test + def test_mandatory_option_backup_files_dir_failure(self): + with tempfile.NamedTemporaryFile() as tmp: + options = {"backup_files_dir": tmp.name} + + with self.assertRaises(CommandError) as exc: + RestoreCommand().validate_backup_file_options(**options) + + self.assertIn("not a directory", exc.exception.args[0], '"not a directory" failure exception expected.') + + # parse_backup_files_dir() method test + def test_backup_files_dir_no_archive(self): + with tempfile.TemporaryDirectory() as tmp: + backup_file = RestoreCommand().parse_backup_files_dir(tmp) + self.assertIsNone(backup_file, "Expecting backup file to be None") + + # parse_backup_files_dir() method test + def test_backup_files_dir_multiple_archives(self): + # create a backup files directory + with tempfile.TemporaryDirectory() as tmp_dir: + # create the 1st backup file + with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file_1: + with zipfile.ZipFile(tmp_file_1, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something1.txt", "Some Content Here") + + # make sure timestamps of files modification differ + time.sleep(1) + + # create the 2nd backup file + with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file_2: + with zipfile.ZipFile(tmp_file_2, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something2.txt", "Some Content Here") + + backup_file = RestoreCommand().parse_backup_files_dir(tmp_dir) + + self.assertIn(tmp_file_2.name, backup_file, "Expected the younger file to be chosen.") + + # parse_backup_files_dir() method test + def test_backup_files_dir_with_newer_restored_backup(self): + # create an entry in restoration history with a dump created in the future + RestoredBackupFactory(creation_date=(datetime.datetime.now()) + datetime.timedelta(days=2)) + + # create a backup files directory + with tempfile.TemporaryDirectory() as tmp_dir: + # create the backup file + with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + backup_file = RestoreCommand().parse_backup_files_dir(tmp_dir) + + self.assertIsNone(backup_file, "Expected the backup not to be chosen.") + + # parse_backup_files_dir() method test + def test_backup_files_dir_with_older_restored_backup(self): + # create an entry in restoration history with a dump created in the past + RestoredBackupFactory(creation_date=(datetime.datetime.now()) - datetime.timedelta(days=2)) + + # create a backup files directory + with tempfile.TemporaryDirectory() as tmp_dir: + # create the backup file + with tempfile.NamedTemporaryFile(dir=tmp_dir) as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + backup_file = RestoreCommand().parse_backup_files_dir(tmp_dir) + + self.assertIn(tmp_file.name, backup_file, "Expected the backup to be chosen.") + + # validate_backup_file_hash() method test + def test_backup_hash_no_md5_file(self): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + file_hash = RestoreCommand().validate_backup_file_hash(tmp_file.name) + + self.assertIsNotNone(file_hash, "Expected the backup file MD5 hash to be returned.") + + # validate_backup_file_hash() method test + def test_backup_hash_failure(self): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + # create a md5 hash file for the backup temporary file + tmp_hash_file = f"{tmp_file.name}.md5" + with open(tmp_hash_file, "w") as hash_file: + hash_file.write("91162629d258a876ee994e9233b2ad87") + + try: + with self.assertRaises(RuntimeError) as exc: + RestoreCommand().validate_backup_file_hash(tmp_file.name) + + self.assertIn( + "Backup archive integrity failure", + exc.exception.args[0], + '"Backup archive integrity failure" exception expected.', + ) + + finally: + # remove temporary hash file + os.remove(tmp_hash_file) + + # validate_backup_file_hash() method test + def test_backup_hash_success(self): + # create the backup file + with tempfile.NamedTemporaryFile() as tmp_file: + with zipfile.ZipFile(tmp_file, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("something.txt", "Some Content Here") + + # create a md5 hash file for the backup temporary file + tmp_hash_file = f"{tmp_file.name}.md5" + with open(tmp_hash_file, "w") as hash_file: + hash_file.write(md5_file_hash(tmp_file.name)) + + try: + file_hash = RestoreCommand().validate_backup_file_hash(tmp_file.name) + + self.assertIsNotNone(file_hash, "Expected the backup file MD5 hash to be returned.") + + finally: + # remove temporary hash file + os.remove(tmp_hash_file) diff --git a/geonode/catalogue/__init__.py b/geonode/catalogue/__init__.py new file mode 100644 index 0000000..a64ed9c --- /dev/null +++ b/geonode/catalogue/__init__.py @@ -0,0 +1,88 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +""" +Tools for managing a Catalogue Service for the Web (CSW) +""" +import os +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from importlib import import_module + + +DEFAULT_CATALOGUE_ALIAS = "default" + +# GeoNode uses this if the CATALOGUE setting is empty (None). +if not hasattr(settings, "CATALOGUE"): + settings.CATALOGUE = {DEFAULT_CATALOGUE_ALIAS: "geonode.backends.dummy"} + +# If settings.CATALOGUE is defined, we expect it to be properly named +if DEFAULT_CATALOGUE_ALIAS not in settings.CATALOGUE: + raise ImproperlyConfigured(f"You must define a '{DEFAULT_CATALOGUE_ALIAS}' CATALOGUE") + + +def load_backend(backend_name): + """load a Catalogue backend""" + # Look for a fully qualified CSW backend name + try: + return import_module(backend_name) + except ImportError as e_user: + # The CSW backend wasn't found. Display a helpful error message + # listing all possible (built-in) CSW backends. + backend_dir = os.path.join(os.path.dirname(__file__), "backends") + try: + available_backends = sorted( + [ + f + for f in os.listdir(backend_dir) + if os.path.isdir(os.path.join(backend_dir, f)) and not f.startswith(".") + ] + ) + except OSError: + available_backends = [] + + if backend_name not in available_backends: + backend_reprs = list(map(repr, available_backends)) + error_msg = ( + f"{backend_name!r} isn't an available catalogue backend.\n" + "Try using geonode.catalogue.backends.BACKEND, where BACKEND is one of:\n" + f" {', '.join(backend_reprs)}\nError was: {e_user}" + ) + raise ImproperlyConfigured(error_msg) + else: + # If there's some other error, this must be an error in GeoNode + raise + + +def default_catalogue_backend(): + """Get the default backend""" + msg = f"There is no '{DEFAULT_CATALOGUE_ALIAS}' backend in CATALOGUE" + assert DEFAULT_CATALOGUE_ALIAS in settings.CATALOGUE, msg + return settings.CATALOGUE[DEFAULT_CATALOGUE_ALIAS] + + +def get_catalogue(backend=None, skip_caps=True): + """Returns a catalogue object.""" + default_backend_config = backend or default_catalogue_backend() + backend_name = default_backend_config["ENGINE"] + catalog_module = load_backend(backend_name) + assert hasattr(catalog_module, "CatalogueBackend"), "%s must define a CatalogueBackend class" + catalog_class = catalog_module.CatalogueBackend + cat = catalog_class(skip_caps=skip_caps, **default_backend_config) + return cat diff --git a/geonode/catalogue/apps.py b/geonode/catalogue/apps.py new file mode 100644 index 0000000..961648a --- /dev/null +++ b/geonode/catalogue/apps.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + + +class GeoNodeCatalogueAppConfig(AppConfig): + name = "geonode.catalogue" + verbose_name = "GeoNode CSW Catalogue" diff --git a/geonode/catalogue/backends/__init__.py b/geonode/catalogue/backends/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/catalogue/backends/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/catalogue/backends/base.py b/geonode/catalogue/backends/base.py new file mode 100644 index 0000000..fafcdfc --- /dev/null +++ b/geonode/catalogue/backends/base.py @@ -0,0 +1,38 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + + +class BaseCatalogueBackend(object): + """Catalogue abstract base class""" + + def remove_record(self, uuid): + """Remove record from the catalogue""" + raise NotImplementedError() + + def create_record(self, item): + """Create record in the catalogue""" + raise NotImplementedError() + + def get_record(self, uuid): + """Get record from the catalogue""" + raise NotImplementedError() + + def search_records(self, keywords, start, limit, bbox): + """Search for records from the catalogue""" + raise NotImplementedError() diff --git a/geonode/catalogue/backends/generic.py b/geonode/catalogue/backends/generic.py new file mode 100644 index 0000000..848fd97 --- /dev/null +++ b/geonode/catalogue/backends/generic.py @@ -0,0 +1,305 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import re +import logging + +from urllib.parse import urlparse, urlencode + +from django.conf import settings +from django.template.loader import get_template +from owslib.catalogue.csw2 import CatalogueServiceWeb, namespaces +from owslib.util import http_post +from owslib.etree import etree as dlxml +from owslib.fes import PropertyIsLike, BBox +from geonode.catalogue.backends.base import BaseCatalogueBackend + +logger = logging.getLogger(__name__) + +TIMEOUT = 10 +METADATA_FORMATS = { + "Atom": ("atom:entry", "http://www.w3.org/2005/Atom"), + "DIF": ("dif:DIF", "http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/"), + "Dublin Core": ("csw:Record", "http://www.opengis.net/cat/csw/2.0.2"), + "ebRIM": ("rim:RegistryObject", "urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0"), + "FGDC": ("fgdc:metadata", "http://www.opengis.net/cat/csw/csdgm"), + "ISO": ("gmd:MD_Metadata", "http://www.isotc211.org/2005/gmd"), +} + + +class Catalogue(CatalogueServiceWeb): + def __init__(self, *args, **kwargs): + self.url = kwargs["URL"] + self.user = None + self.password = None + self.type = kwargs["ENGINE"].split(".")[-1] + self.local = False + self._group_ids = {} + self._operation_ids = {} + self.connected = False + skip_caps = kwargs.get("skip_caps", True) + CatalogueServiceWeb.__init__(self, url=self.url, skip_caps=skip_caps) + + upurl = urlparse(self.url) + + self.base = f"{upurl.scheme}://{upurl.netloc}/" + + # User and Password are optional + if "USER" in kwargs: + self.user = kwargs["USER"] + if "PASSWORD" in kwargs: + self.password = kwargs["PASSWORD"] + + def __enter__(self, *args, **kwargs): + self.login() + return self + + def __exit__(self, *args, **kwargs): + self.logout() + + def login(self): + NotImplemented + + def logout(self): + NotImplemented + + def get_by_uuid(self, uuid): + try: + self.getrecordbyid([uuid], outputschema=namespaces["gmd"]) + except Exception: + return None + + if hasattr(self, "records"): + if len(self.records) < 1: + return None + record = list(self.records.values())[0] + record.keywords = [] + if ( + hasattr(record, "identification") + and len(record.identification) > 0 + and hasattr(record.identification[0], "keywords") + ): + for kw in record.identification[0].keywords: + record.keywords.extend(kw.keywords) + return record + else: + return None + + def url_for_uuid(self, uuid, outputschema): + _query_string = urlencode( + { + "request": "GetRecordById", + "service": "CSW", + "version": "2.0.2", + "id": uuid, + "outputschema": outputschema, + "elementsetname": "full", + } + ) + return f"{self.url}?{_query_string}" + + def urls_for_uuid(self, uuid): + """returns list of valid GetRecordById URLs for a given record""" + urls = [] + for mformat in self.formats: + urls.append(("text/xml", mformat, self.url_for_uuid(uuid, METADATA_FORMATS[mformat][1]))) + return urls + + def csw_gen_xml(self, layer, template): + id_pname = "dc:identifier" + if self.type == "deegree": + id_pname = "apiso:Identifier" + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + tpl = get_template(template) + ctx = { + "CATALOG_METADATA_TEMPLATE": settings.CATALOG_METADATA_TEMPLATE, + "layer": layer, + "SITEURL": site_url, + "id_pname": id_pname, + "LICENSES_METADATA": getattr(settings, "LICENSES", dict()).get("METADATA", "never"), + } + md_doc = tpl.render(context=ctx) + return md_doc + + def csw_gen_anytext(self, xml): + """get all element data from an XML document""" + xml = dlxml.fromstring(xml) + return " ".join([value.strip() for value in xml.xpath("//text()")]) + + def csw_request(self, layer, template): + md_doc = self.csw_gen_xml(layer, template) + response = http_post(self.url, md_doc, timeout=TIMEOUT) + return response + + def create_from_dataset(self, layer): + response = self.csw_request(layer, "catalogue/transaction_insert.xml") # noqa + # TODO: Parse response, check for error report + return self.url_for_uuid(layer.uuid, namespaces["gmd"]) + + def delete_dataset(self, layer): + response = self.csw_request(layer, "catalogue/transaction_delete.xml") # noqa + # TODO: Parse response, check for error report + + def update_dataset(self, layer): + tmpl = "catalogue/transaction_update.xml" + response = self.csw_request(layer, tmpl) # noqa + # TODO: Parse response, check for error report + + def search(self, keywords, startposition, maxrecords, bbox): + """CSW search wrapper""" + formats = [] + for f in self.formats: + formats.append(METADATA_FORMATS[f][0]) + + dataset_query_like = [] + if keywords: + for _kw in keywords: + dataset_query_like.append(PropertyIsLike("csw:AnyText", _kw)) + bbox_query = [] + if bbox: + bbox_query = BBox(bbox) + return self.getrecords2( + typenames=" ".join(formats), + constraints=dataset_query_like + bbox_query, + startposition=startposition, + maxrecords=maxrecords, + outputschema="http://www.isotc211.org/2005/gmd", + esn="full", + ) + + def normalize_bbox(self, bbox): + return [bbox[1], bbox[0], bbox[3], bbox[2]] + + def metadatarecord2dict(self, rec): + """ + accepts a node representing a catalogue result + record and builds a POD structure representing + the search result. + """ + + if rec is None: + return None + # Let owslib do some parsing for us... + result = {} + result["uuid"] = rec.identifier + result["title"] = rec.identification[0].title + result["abstract"] = rec.identification[0].abstract + + keywords = [] + for kw in rec.identification[0].keywords: + keywords.extend(kw["keywords"]) + + result["keywords"] = keywords + + # XXX needs indexing ? how + result["attribution"] = {"title": "", "href": ""} + + result["name"] = result["uuid"] + + result["bbox"] = { + "minx": rec.identification[0].bbox.minx, + "maxx": rec.identification[0].bbox.maxx, + "miny": rec.identification[0].bbox.miny, + "maxy": rec.identification[0].bbox.maxy, + } + + # locate all distribution links + result["download_links"] = self.extract_links(rec) + + # construct the link to the Catalogue metadata record (not + # self-indexed) + result["metadata_links"] = [ + ("text/xml", "ISO", self.url_for_uuid(rec.identifier, "http://www.isotc211.org/2005/gmd")) + ] + + return result + + def extract_links(self, rec): + # fetch all distribution links + + links = [] + # extract subset of description value for user-friendly display + format_re = re.compile(r".*\((.*)(\s*Format*\s*)\).*?") + + if not hasattr(rec, "distribution"): + return None + if not hasattr(rec.distribution, "online"): + return None + + for link_el in rec.distribution.online: + if "WWW:DOWNLOAD" in link_el.protocol: + try: + extension = link_el.name.split(".")[-1] + format = format_re.match(link_el.description).groups()[0] + href = link_el.url + links.append((extension, format, href)) + except Exception: + pass + return links + + +class CatalogueBackend(BaseCatalogueBackend): + def __init__(self, *args, **kwargs): + self.catalogue = Catalogue(*args, **kwargs) + + def get_record(self, uuid): + with self.catalogue: + rec = self.catalogue.get_by_uuid(uuid) + if rec is not None: + rec.links = dict() + rec.links["metadata"] = self.catalogue.urls_for_uuid(uuid) + rec.links["download"] = self.catalogue.extract_links(rec) + return rec + + def search_records(self, keywords, start, limit, bbox): + with self.catalogue: + bbox = self.catalogue.normalize_bbox(bbox) + self.catalogue.search(keywords, start + 1, limit, bbox) + + # build results into JSON for API + results = [self.catalogue.metadatarecord2dict(doc) for v, doc in self.catalogue.records.items()] + + result = { + "rows": results, + "total": self.catalogue.results["matches"], + "next_page": self.catalogue.results.get("nextrecord", 0), + } + + return result + + def remove_record(self, uuid): + with self.catalogue: + catalogue_record = self.catalogue.get_by_uuid(uuid) + if catalogue_record is None: + return + try: + # this is a bit hacky, delete_dataset expects an instance of the layer + # model but it just passes it to a Django template so a dict works + # too. + self.catalogue.delete_dataset({"uuid": uuid}) + except Exception: + logger.exception("Couldn't delete Catalogue record during cleanup()") + + def create_record(self, item): + with self.catalogue: + record = self.catalogue.get_by_uuid(item.uuid) + if record is None: + md_link = self.catalogue.create_from_dataset(item) + item.metadata_links = [("text/xml", "ISO", md_link)] + else: + self.catalogue.update_dataset(item) diff --git a/geonode/catalogue/backends/pycsw_http.py b/geonode/catalogue/backends/pycsw_http.py new file mode 100644 index 0000000..0058c41 --- /dev/null +++ b/geonode/catalogue/backends/pycsw_http.py @@ -0,0 +1,29 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.catalogue.backends.generic import CatalogueBackend as GenericCatalogueBackend + + +class CatalogueBackend(GenericCatalogueBackend): + """pycsw HTTP CSW backend""" + + def __init__(self, *args, **kwargs): + """initialize pycsw HTTP CSW backend""" + GenericCatalogueBackend.__init__(CatalogueBackend, self, *args, **kwargs) + self.catalogue.formats = ["Atom", "DIF", "Dublin Core", "ebRIM", "FGDC", "ISO"] diff --git a/geonode/catalogue/backends/pycsw_local.py b/geonode/catalogue/backends/pycsw_local.py new file mode 100644 index 0000000..75c66e9 --- /dev/null +++ b/geonode/catalogue/backends/pycsw_local.py @@ -0,0 +1,178 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +from owslib.etree import etree as dlxml +from django.conf import settings +from owslib.iso import MD_Metadata +from pycsw import server +from geonode.catalogue.backends.generic import CatalogueBackend as GenericCatalogueBackend +from geonode.catalogue.backends.generic import METADATA_FORMATS +from shapely.errors import WKBReadingError, WKTReadingError + +true_value = "true" +false_value = "false" +if settings.DATABASES["default"]["ENGINE"].endswith( + ( + "sqlite", + "sqlite3", + "spatialite", + ) +): + true_value = "1" + false_value = "0" + +# pycsw settings that the user shouldn't have to worry about +CONFIGURATION = { + "server": { + "home": ".", + "url": settings.CATALOGUE["default"]["URL"], + "encoding": "UTF-8", + "language": settings.LANGUAGE_CODE, + "maxrecords": "10", + # 'loglevel': 'DEBUG', + # 'logfile': '/tmp/pycsw.log', + # 'federatedcatalogues': 'http://geo.data.gov/geoportal/csw/discovery', + "pretty_print": "true", + "domainquerytype": "range", + "domaincounts": "true", + "profiles": "apiso,ebrim", + }, + "repository": { + "source": "geonode.catalogue.backends.pycsw_plugin.GeoNodeRepository", + "filter": "uuid IS NOT NULL", + "mappings": os.path.join(os.path.dirname(__file__), "pycsw_local_mappings.py"), + }, +} + + +class CatalogueBackend(GenericCatalogueBackend): + def __init__(self, *args, **kwargs): + GenericCatalogueBackend.__init__(CatalogueBackend, self, *args, **kwargs) + self.catalogue.formats = ["Atom", "DIF", "Dublin Core", "ebRIM", "FGDC", "ISO"] + self.catalogue.local = True + + def remove_record(self, uuid): + pass + + def create_record(self, item): + pass + + def get_record(self, uuid): + results = self._csw_local_dispatch(identifier=uuid) + if len(results) < 1: + return None + + result = dlxml.fromstring(results).find("{http://www.isotc211.org/2005/gmd}MD_Metadata") + + if result is None: + return None + + record = MD_Metadata(result) + record.keywords = [] + if hasattr(record, "identification") and hasattr(record.identification[0], "keywords"): + for kw in record.identification[0].keywords: + record.keywords.extend([_kw.name for _kw in kw.keywords]) + + record.links = {} + record.links["metadata"] = self.catalogue.urls_for_uuid(uuid) + record.links["download"] = self.catalogue.extract_links(record) + return record + + def search_records(self, keywords, start, limit, bbox): + with self.catalogue: + lresults = self._csw_local_dispatch(keywords, keywords, start + 1, limit, bbox) + # serialize XML + e = dlxml.fromstring(lresults) + + self.catalogue.records = [ + MD_Metadata(x) for x in e.findall("//{http://www.isotc211.org/2005/gmd}MD_Metadata") + ] + + # build results into JSON for API + results = [self.catalogue.metadatarecord2dict(doc) for v, doc in self.catalogue.records.items()] + + result = { + "rows": results, + "total": e.find("{http://www.opengis.net/cat/csw/2.0.2}SearchResults").attrib.get( + "numberOfRecordsMatched" + ), + "next_page": e.find("{http://www.opengis.net/cat/csw/2.0.2}SearchResults").attrib.get("nextRecord"), + } + + return result + + def _csw_local_dispatch(self, keywords=None, start=0, limit=10, bbox=None, identifier=None): + """ + HTTP-less CSW + """ + + mdict = dict(settings.PYCSW["CONFIGURATION"], **CONFIGURATION) + if "server" in settings.PYCSW["CONFIGURATION"]: + # override server system defaults with user specified directives + mdict["server"].update(settings.PYCSW["CONFIGURATION"]["server"]) + + # fake HTTP environment variable + os.environ["QUERY_STRING"] = "" + + # init pycsw + csw = server.Csw(mdict, version="2.0.2") + + # fake HTTP method + csw.requesttype = "GET" + + # fake HTTP request parameters + if identifier is None: # it's a GetRecords request + formats = [] + for f in self.catalogue.formats: + formats.append(METADATA_FORMATS[f][0]) + + csw.kvp = { + "service": "CSW", + "version": "2.0.2", + "elementsetname": "full", + "typenames": formats, + "resulttype": "results", + "constraintlanguage": "CQL_TEXT", + "outputschema": "http://www.isotc211.org/2005/gmd", + "constraint": None, + "startposition": start, + "maxrecords": limit, + } + response = csw.getrecords2() + else: # it's a GetRecordById request + csw.kvp = { + "service": "CSW", + "version": "2.0.2", + "request": "GetRecordById", + "id": identifier, + "outputschema": "http://www.isotc211.org/2005/gmd", + } + # FIXME(Ariel): Remove this try/except block when pycsw deals with + # empty geometry fields better. + # https://gist.github.com/ingenieroariel/717bb720a201030e9b3a + try: + response = csw.dispatch() + except (WKBReadingError, WKTReadingError): + return [] + + if isinstance(response, list): # pycsw 2.0+ + response = response[1] + + return response diff --git a/geonode/catalogue/backends/pycsw_local_mappings.py b/geonode/catalogue/backends/pycsw_local_mappings.py new file mode 100644 index 0000000..4bbb864 --- /dev/null +++ b/geonode/catalogue/backends/pycsw_local_mappings.py @@ -0,0 +1,83 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +# based on https://github.com/geopython/pycsw/blob/master/pycsw/core/config.py +MD_CORE_MODEL = { + "typename": "pycsw:CoreMetadata", + "outputschema": "http://pycsw.org/metadata", + "mappings": { + "pycsw:Identifier": "uuid", + "pycsw:Typename": "csw_typename", + "pycsw:Schema": "csw_schema", + "pycsw:MdSource": "csw_mdsource", + "pycsw:InsertDate": "csw_insert_date", + "pycsw:XML": "metadata_xml", + "pycsw:AnyText": "csw_anytext", + "pycsw:Language": "language", + "pycsw:Title": "title", + "pycsw:Abstract": "raw_abstract", + "pycsw:Keywords": "keyword_csv", + "pycsw:KeywordType": "keywordstype", + "pycsw:Format": "spatial_representation_type_string", + "pycsw:Source": "source", + "pycsw:Date": "date", + "pycsw:Modified": "date", + "pycsw:Type": "csw_type", + "pycsw:BoundingBox": "csw_wkt_geometry", + "pycsw:CRS": "csw_crs", + "pycsw:AlternateTitle": "alternate", + "pycsw:RevisionDate": "date", + "pycsw:CreationDate": "date", + "pycsw:PublicationDate": "date", + "pycsw:Organization": "organizationname", + "pycsw:OrganizationName": "organizationname", + "pycsw:SecurityConstraints": "securityconstraints", + "pycsw:ParentIdentifier": "parentidentifier", + "pycsw:TopicCategory": "topiccategory", + "pycsw:ResourceLanguage": "language", + "pycsw:GeographicDescriptionCode": "geodescode", + "pycsw:Denominator": "denominator", + "pycsw:DistanceValue": "distancevalue", + "pycsw:DistanceUOM": "distanceuom", + "pycsw:TempExtent_begin": "temporal_extent_start", + "pycsw:TempExtent_end": "temporal_extent_end", + "pycsw:ServiceType": "servicetype", + "pycsw:ServiceTypeVersion": "servicetypeversion", + "pycsw:Operation": "operation", + "pycsw:CouplingType": "couplingtype", + "pycsw:OperatesOn": "operateson", + "pycsw:OperatesOnIdentifier": "operatesonidentifier", + "pycsw:OperatesOnName": "operatesoname", + "pycsw:Degree": "degree", + "pycsw:AccessConstraints": "restriction_code", + "pycsw:OtherConstraints": "raw_constraints_other", + "pycsw:Classification": "classification", + "pycsw:ConditionApplyingToAccessAndUse": "conditionapplyingtoaccessanduse", + "pycsw:Lineage": "lineage", + "pycsw:ResponsiblePartyRole": "responsiblepartyrole", + "pycsw:SpecificationTitle": "specificationtitle", + "pycsw:SpecificationDate": "specificationdate", + "pycsw:SpecificationDateType": "specificationdatetype", + "pycsw:Creator": "creator", + "pycsw:Publisher": "publisher", + "pycsw:Contributor": "contributor", + "pycsw:Relation": "relation", + "pycsw:Links": "download_links", + }, +} diff --git a/geonode/catalogue/backends/pycsw_plugin.py b/geonode/catalogue/backends/pycsw_plugin.py new file mode 100644 index 0000000..7c951b2 --- /dev/null +++ b/geonode/catalogue/backends/pycsw_plugin.py @@ -0,0 +1,211 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.db import connection +from django.db.models import Max, Min, Count +from django.conf import settings + +from pycsw.core.repository import Repository, query_spatial, get_geometry_area + +from geonode.base.models import ResourceBase + +from pycsw.core import util + +LOGGER = logging.getLogger(__name__) + +GEONODE_SERVICE_TYPES = { + # 'GeoNode enum': 'CSW enum' + "http://www.opengis.net/cat/csw/2.0.2": "OGC:CSW", + "http://www.opengis.net/wms": "OGC:WMS", + "http://www.opengis.net/wmts/1.0": "OGC:WMTS", + "https://wiki.osgeo.org/wiki/TMS": "OSGeo:TMS", + "urn:x-esri:serviceType:ArcGIS:MapServer": "ESRI:ArcGIS:MapServer", + "urn:x-esri:serviceType:ArcGIS:ImageServer": "ESRI:ArcGIS:ImageServer", +} + + +class GeoNodeRepository(Repository): + """ + Class to interact with underlying repository + """ + + def __init__(self, context, repo_filter=None): + """ + Initialize repository + """ + + self.context = context + self.filter = repo_filter + self.fts = False + self.label = "GeoNode" + self.local_ingest = True + + self.dbtype = settings.DATABASES["default"]["ENGINE"].split(".")[-1] + + # GeoNode PostgreSQL installs are PostGIS enabled + if self.dbtype == "postgis": + self.dbtype = "postgresql+postgis+wkt" + + if self.dbtype in {"sqlite", "sqlite3"}: # load SQLite query bindings + connection.connection.create_function("query_spatial", 4, query_spatial) + connection.connection.create_function("get_anytext", 1, util.get_anytext) + connection.connection.create_function("get_geometry_area", 1, get_geometry_area) + + # generate core queryables db and obj bindings + self.queryables = {} + + for tname in self.context.model["typenames"]: + for qname in self.context.model["typenames"][tname]["queryables"]: + self.queryables[qname] = {} + items = list(self.context.model["typenames"][tname]["queryables"][qname].items()) + + for qkey, qvalue in items: + self.queryables[qname][qkey] = qvalue + + # flatten all queryables + # TODO smarter way of doing this + self.queryables["_all"] = {} + for qbl in self.queryables: + self.queryables["_all"].update(self.queryables[qbl]) + self.queryables["_all"].update(self.context.md_core_model["mappings"]) + + if "Harvest" in self.context.model["operations"] and "Transaction" in self.context.model["operations"]: + self.context.model["operations"]["Harvest"]["parameters"]["ResourceType"]["values"] = list( + GEONODE_SERVICE_TYPES.keys() + ) # noqa + self.context.model["operations"]["Transaction"]["parameters"]["TransactionSchemas"]["values"] = list( + GEONODE_SERVICE_TYPES.keys() + ) # noqa + + def dataset(self): + """ + Stub to mock a pycsw dataset object for Transactions + """ + return type("ResourceBase", (object,), {}) + + def query_ids(self, ids): + """ + Query by list of identifiers + """ + + results = self._get_repo_filter(ResourceBase.objects).filter(uuid__in=ids).all() + + return results + + def query_domain(self, domain, typenames, domainquerytype="list", count=False): + """ + Query by property domain values + """ + + objects = self._get_repo_filter(ResourceBase.objects) + + if domainquerytype == "range": + return [tuple(objects.aggregate(Min(domain), Max(domain)).values())] + else: + if count: + return [(d[domain], d[f"{domain}__count"]) for d in objects.values(domain).annotate(Count(domain))] + else: + return objects.values_list(domain).distinct() + + def query_insert(self, direction="max"): + """ + Query to get latest (default) or earliest update to repository + """ + if direction == "min": + return ResourceBase.objects.aggregate(Min("last_updated"))["last_updated__min"].strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + return ( + self._get_repo_filter(ResourceBase.objects) + .aggregate(Max("last_updated"))["last_updated__max"] + .strftime("%Y-%m-%dT%H:%M:%SZ") + ) + + def query_source(self, source): + """ + Query by source + """ + return self._get_repo_filter(ResourceBase.objects).filter(url=source) + + def query(self, constraint, sortby=None, typenames=None, maxrecords=10, startposition=0): + """ + Query records from underlying repository + """ + + # run the raw query and get total + # we want to exclude layers which are not valid, as it is done in the + # search engine + pycsw_filters = settings.PYCSW.get("FILTER", {"resource_type__in": ["dataset"]}) + + if "where" in constraint: # GetRecords with constraint + query = self._get_repo_filter(ResourceBase.objects.filter(**pycsw_filters)).extra( + where=[constraint["where"]], params=constraint["values"] + ) + else: # GetRecords sans constraint + query = self._get_repo_filter(ResourceBase.objects.filter(**pycsw_filters)) + + total = query.count() + + # apply sorting, limit and offset + if sortby is not None: + if "spatial" in sortby and sortby["spatial"]: # spatial sort + desc = False + if sortby["order"] == "DESC": + desc = True + query = query.all() + return [ + str(total), + sorted( + query, + key=lambda x: float(util.get_geometry_area(getattr(x, sortby["propertyname"]))), + reverse=desc, + )[startposition : startposition + int(maxrecords)], + ] + else: + if sortby["order"] == "DESC": + pname = f"-{sortby['propertyname']}" + else: + pname = sortby["propertyname"] + return [str(total), query.order_by(pname)[startposition : startposition + int(maxrecords)]] + else: # no sort + return [str(total), query.all()[startposition : startposition + int(maxrecords)]] + + def delete(self, constraint): + """ + Delete a record from the repository + """ + + results = ( + self._get_repo_filter(ResourceBase.objects) + .extra(where=[constraint["where"]], params=constraint["values"]) + .all() + ) + deleted = len(results) + results.delete() + return deleted + + def _get_repo_filter(self, query): + """ + Apply repository wide side filter / mask query + """ + if self.filter is not None: + return query.extra(where=[self.filter]) + return query diff --git a/geonode/catalogue/backends/tests.py b/geonode/catalogue/backends/tests.py new file mode 100644 index 0000000..f579b68 --- /dev/null +++ b/geonode/catalogue/backends/tests.py @@ -0,0 +1,58 @@ +import ast +from django.test.utils import override_settings +from owslib.etree import etree +from geonode.base.populate_test_data import create_single_doc, create_single_dataset, create_single_map +from django.contrib.auth.models import AnonymousUser +from django.test.client import RequestFactory +from geonode.catalogue.views import csw_global_dispatch +from django.test import TestCase +from django.conf import settings + +pycsw_settings = settings.PYCSW.copy() +pycsw_settings_all = settings.PYCSW.copy() +pycsw_settings["FILTER"] = {"resource_type__in": ["dataset", "map"]} +pycsw_settings_all["FILTER"] = {"resource_type__in": ["dataset", "map", "document"]} + + +class TestGeoNodeRepository(TestCase): + # to simplify the tests we pass throught csw_global_dispatch + # since call the GeoNodeRepository.query + def setUp(self): + self.layer = create_single_dataset("dataset_name") + self.map = create_single_map("map_name") + self.doc = create_single_doc("doc_name") + self.request = self.__request_factory() + + def test_if_pycsw_filter_is_not_set_should_return_only_the_dataset_by_default(self): + response = csw_global_dispatch(self.request) + root = etree.fromstring(response.content) + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 + self.assertEqual(1, returned_results) + + @override_settings(PYCSW=pycsw_settings) + def test_if_pycsw_filter_is_set_should_return_only_datasets_and_map(self): + response = csw_global_dispatch(self.request) + root = etree.fromstring(response.content) + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 + self.assertEqual(2, returned_results) + + @override_settings(PYCSW=pycsw_settings_all) + def test_if_pycsw_filter_is_set_should_return_all_datasets_map_doc(self): + response = csw_global_dispatch(self.request) + root = etree.fromstring(response.content) + child = [x.attrib for x in root if "numberOfRecordsMatched" in x.attrib] + returned_results = ast.literal_eval(child[0].get("numberOfRecordsMatched", "0")) if child else 0 + self.assertEqual(3, returned_results) + + @staticmethod + def __request_factory(): + factory = RequestFactory() + url = "http://localhost:8000/catalogue/csw?request=GetRecords" + url += "&service=CSW&version=2.0.2&outputschema=http%3A%2F%2Fwww.isotc211.org%2F2005%2Fgmd" + url += "&elementsetname=brief&typenames=csw:Record&resultType=results" + request = factory.get(url) + + request.user = AnonymousUser() + return request diff --git a/geonode/catalogue/management/__init__.py b/geonode/catalogue/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/catalogue/management/commands/__init__.py b/geonode/catalogue/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/catalogue/management/commands/regenerate_xml.py b/geonode/catalogue/management/commands/regenerate_xml.py new file mode 100644 index 0000000..ce89abd --- /dev/null +++ b/geonode/catalogue/management/commands/regenerate_xml.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.core.management.base import BaseCommand + +from geonode.base.management import command_utils +from geonode.base.models import ResourceBase +from geonode.layers.models import Dataset + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Re-create XML metadata documents" + + def add_arguments(self, parser): + parser.add_argument( + '-l', + '--layer', + dest="layers", + action='append', + help="Only process specified layers ") + + parser.add_argument( + "--skip-logger-setup", + action="store_false", + dest="setup_logger", + help='Skips setup of the "geonode.br" logger, "br" handler and "br" format if not present in settings', + ) + parser.add_argument( + '-d', + '--dry-run', + dest="dry-run", + action='store_true', + help="Do not actually perform any change") + + def handle(self, **options): + requested_layers = options.get('layers') + dry_run = options.get('dry-run') + + if options.get("setup_logger"): + logger = command_utils.setup_logger() + + logger.info(f"==== Running command {__name__}") + logger.info(f"{self.help}") + logger.info("") + + logger.debug(f"DRY-RUN is {dry_run}") + logger.debug(f"LAYERS is {requested_layers}") + + try: + + layers = Dataset.objects.all() + tot = len(layers) + logger.info(f"Total layers in GeoNode: {tot}") + i = 0 + cnt_ok = 0 + cnt_bad = 0 + cnt_skip = 0 + + instance: ResourceBase + for instance in layers: + i += 1 + logger.info(f"- {i}/{tot} Processing layer {instance.id} [{instance.typename}] '{instance.title}'") + + if requested_layers and instance.typename not in requested_layers: + logger.info(" - Layer filtered out by args") + cnt_skip += 1 + continue + + if instance.metadata_uploaded and instance.metadata_uploaded_preserve: + logger.info(" - Layer filtered out since it uses custom XML") + cnt_skip += 1 + continue + + try: + good = None + if not dry_run: + try: + try: + # the save() method triggers the metadata regeneration + instance.save() + good = True + except Exception as e: + logger.error(f"Error saving instance '{instance.title}': {e}") + raise e + + except Exception as e: + logger.exception(f"Error processing '{instance.title}': {e}", e) + + if dry_run or good: + logger.info(f" - Done {instance.name}") + cnt_ok += 1 + else: + logger.warning(f"Metadata couldn't be regenerated for instance '{instance.title}' ") + cnt_bad += 1 + + except Exception as e: + raise e + except Exception as e: + raise e + + logger.info("Work completed" + (" [DRYRUN]" if dry_run else "")) + logger.info(f"- Metadata regenerated : {cnt_ok}") + logger.info(f"- Metadata in error : {cnt_bad}") + logger.info(f"- Resources skipped : {cnt_skip}") diff --git a/geonode/catalogue/metadataxsl/README.md b/geonode/catalogue/metadataxsl/README.md new file mode 100644 index 0000000..2d29b1d --- /dev/null +++ b/geonode/catalogue/metadataxsl/README.md @@ -0,0 +1,42 @@ +# Metadata XSL output App + +The ``metadataxsl`` contrib app provides a downloadable metadata having the same format as the ISO +metadata, but having a header like +```XML + +``` +in order to have the metadata properly formatted on the browser. + +The interface will present the link "ISO with XSL" along with the other output formats: + ![image](https://cloud.githubusercontent.com/assets/717359/14913848/4663a80c-0e06-11e6-868d-0877acdb65d6.png) + +In order to have a different output, you will only need to customize the XSL file and provide the CSS files if needed. + +# Settings + +## Update the existing resources + +In order to make it possibile for a resource to have its metadata exported as ISO+XSL, it needs to +have a new entry in the ``base_link`` table. +The implementation will create the needed links every time a new resource is added, but we need to +post-process the existing resources and add the missing links. + +In order to add the missing links, you need to enter the geonode directory and issue the command: + + python manage.py addmissinglinks + +For each resource having a link for the ISO metadata, a new link for the ISO+XSL will be created. + +## Collect static files + +This step is needed in order to copy in the static dir the ``.xsl`` file that will be referenced by the +exported metadata file, and one ``.css`` file that is referenced within the xsl file. + + python manage.py collectstatic + +This means that any customization to the output format should be performed on these files. + +# Other refs + +Please check the issue https://github.com/GeoNode/geonode/issues/2453 diff --git a/geonode/catalogue/metadataxsl/__init__.py b/geonode/catalogue/metadataxsl/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/catalogue/metadataxsl/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/catalogue/metadataxsl/apps.py b/geonode/catalogue/metadataxsl/apps.py new file mode 100644 index 0000000..af8e4d6 --- /dev/null +++ b/geonode/catalogue/metadataxsl/apps.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + + +class GeoNodeCatalogueMetadataxslAppConfig(AppConfig): + name = "geonode.catalogue.metadataxsl" + verbose_name = "GeoNode Catalogue metadataxsl" diff --git a/geonode/catalogue/metadataxsl/management/__init__.py b/geonode/catalogue/metadataxsl/management/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/catalogue/metadataxsl/management/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/catalogue/metadataxsl/management/commands/__init__.py b/geonode/catalogue/metadataxsl/management/commands/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/catalogue/metadataxsl/management/commands/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py b/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py new file mode 100644 index 0000000..919cfda --- /dev/null +++ b/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py @@ -0,0 +1,42 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand + +from geonode.base.models import Link, ResourceBase +from geonode.catalogue.metadataxsl.models import add_xsl_link + + +class Command(BaseCommand): + help = 'Add missing links to ISO XSL metadata service' + + def handle(self, *args, **options): + + for resource in ResourceBase.objects.all(): + print(f'Checking resource with id {resource.id}') + + # check ISO link exists + isolink = Link.objects.filter(resource_id=resource.id, link_type='metadata', name='ISO') + if isolink: + print(f' ISO link found for resource {resource.id} "{resource.title}"') + created = add_xsl_link(resource) + if created: + print(' XSL link created') + else: + print(f' ISO link NOT found for resource {resource.id} "{resource.title}"') diff --git a/geonode/catalogue/metadataxsl/models.py b/geonode/catalogue/metadataxsl/models.py new file mode 100644 index 0000000..56b06ba --- /dev/null +++ b/geonode/catalogue/metadataxsl/models.py @@ -0,0 +1,60 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from urllib.parse import urljoin + +from django.conf import settings +from django.urls import reverse +from django.db.models import signals + +from geonode.base.models import Link +from geonode.layers.models import Dataset +from geonode.documents.models import Document + + +ISO_XSL_NAME = "ISO with XSL" + +settings.DOWNLOAD_FORMATS_METADATA.append(ISO_XSL_NAME) + + +def xsl_post_save(instance, sender, **kwargs): + """Add a link to the enriched ISO metadata""" + add_xsl_link(instance.resourcebase_ptr) + + +def add_xsl_link(resourcebase): + """Add a link to the enriched ISO metadata""" + try: + urlpath = reverse("prefix_xsl_line", args=[resourcebase.id]) + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + url = urljoin(site_url, urlpath) + link, created = Link.objects.get_or_create( + resource=resourcebase, + url=url, + defaults=dict(name=ISO_XSL_NAME, extension="xml", mime="text/xml", link_type="metadata"), + ) + return created + except Exception: + return False + + +if "geonode.catalogue" in settings.INSTALLED_APPS: + signals.post_save.connect(xsl_post_save, sender=Dataset) + signals.post_save.connect(xsl_post_save, sender=Document) + # TODO: maps as well? diff --git a/geonode/catalogue/metadataxsl/static/metadataxsl/metadata.css b/geonode/catalogue/metadataxsl/static/metadataxsl/metadata.css new file mode 100644 index 0000000..7701ad3 --- /dev/null +++ b/geonode/catalogue/metadataxsl/static/metadataxsl/metadata.css @@ -0,0 +1,185 @@ +table a:link { + color: #666; + font-weight: bold; + text-decoration:none; +} +table a:visited { + color: #999999; + font-weight:bold; + text-decoration:none; +} +table a:active, +table a:hover { + color: #bd5a35; + text-decoration:underline; +} +table { + font-family:Arial, sans-serif; + color:#666; + font-size:12px; + text-shadow: 1px 1px 0px #fff; + background:#eaebec; + margin:10px; + border:#ccc 1px solid; + width:98%; + -moz-border-radius:3px; + -webkit-border-radius:3px; + border-radius:3px; + -moz-box-shadow: 0 1px 2px #d1d1d1; + -webkit-box-shadow: 0 1px 2px #d1d1d1; + box-shadow: 0 1px 2px #d1d1d1; +} +table th { + padding:10px 10px 10px 10px; + border-top:1px solid #fafafa; + border-bottom:1px solid #e0e0e0; + background: #ededed; + background: -webkit-gradient(linear, left top, left bottom, from(#ededed), to(#ebebeb)); + background: -moz-linear-gradient(top, #ededed, #ebebeb); +} +table th:first-child { + text-align: left; + padding-left:10px; +} +table tr:first-child th:first-child { + -moz-border-radius-topleft:3px; + -webkit-border-top-left-radius:3px; + border-top-left-radius:3px; +} +table tr:first-child th:last-child { + -moz-border-radius-topright:3px; + -webkit-border-top-right-radius:3px; + border-top-right-radius:3px; +} +table tr { + padding-left:10px; +} +table td:first-child { + text-align: left; + padding-left:10px; + border-left: 0; +} +table td { + padding:10px; + border-top: 1px solid #ffffff; + border-bottom:1px solid #e0e0e0; + border-left: 1px solid #e0e0e0; + + background: #fafafa; + background: -webkit-gradient(linear, left top, left bottom, from(#fbfbfb), to(#fafafa)); + background: -moz-linear-gradient(top, #fbfbfb, #fafafa); +} +table tr.even td { + background: #f6f6f6; + background: -webkit-gradient(linear, left top, left bottom, from(#f8f8f8), to(#f6f6f6)); + background: -moz-linear-gradient(top, #f8f8f8, #f6f6f6); +} +table tr:last-child td { + border-bottom:0; +} +table tr:last-child td:first-child { + -moz-border-radius-bottomleft:3px; + -webkit-border-bottom-left-radius:3px; + border-bottom-left-radius:3px; +} +table tr:last-child td:last-child { + -moz-border-radius-bottomright:3px; + -webkit-border-bottom-right-radius:3px; + border-bottom-right-radius:3px; +} +table td img { + vertical-align: middle; + margin-right:10px; +} + +// General styles +body { + padding: 0 2em; + font-family: Arial, sans-serif; + color: #0B85C3; + background : #f2f2f2; +} + +h1, h2, h3 { + font-family: Arial, sans-serif; + font-weight: bold; + color: #0B85C3; + span {color: #167F92;}; +} + +p, table { + font-family: Arial, sans-serif; + font-weight: normal; + font-size:95%; + span {color: #167F92;} +} + +p { + margin:0; padding:0; + text-align:justify; +} + +.header { + background: #efefef; +} + +.title { + color: #0B85C3; + font-weight: bold; +} + +.nowrap { + white-space: nowrap; +} + +.logos { + width:99%; + display:float; + vertical-align:bottom; + clear:both; +} + +.fleft { + float:left; + padding:5px; +} + +.fright { + float:right; + margin:15px 10px; +} + +.pad { + padding-left:25px; +} + + +.fixedcol { + width:300px; +} + +.geonodelogo { + background-color: #1177A3; +} + +@media print { + body { + font-size:90%; + } + .fleft { + width: 30% !important; + height: 30% !important; + margin:0px; + } + .fright { + width: 10% !important; + height: 10% !important; + margin:0px; + } + .pad { + padding-left:20px; + } + .fixedcol { + width:100px; + } +} diff --git a/geonode/catalogue/metadataxsl/static/metadataxsl/metadata.xsl b/geonode/catalogue/metadataxsl/static/metadataxsl/metadata.xsl new file mode 100644 index 0000000..017708d --- /dev/null +++ b/geonode/catalogue/metadataxsl/static/metadataxsl/metadata.xsl @@ -0,0 +1,1137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    + + + + + + + + + + +
    + + +

    + +

    +     + + +

    + + Abstract: + + +

    +     +
    + + +

    + +

    +     + + +

    + + Abstract: + + +

    +     +
    +     + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Hierarchy level + + + + + + + + + + + + + + + 150 + 150 + this.onerror=null;this.src='/static/geonode/img/missing_thumb.png' + + + +
    + Resource Identifier + + + | + +
    + Alternative Title + + + + +
    + Resource Status + + + + +
    +     +     +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    WHAT

    +
    + Resource Purpose + + + + +
    + Spatial Representation Type + + + +
    +     +
    + Topic Category + + + + + Farming
    + Biota
    + Boundaries
    + Climatology, meteorology, atmosphere
    + Economy
    + Elevation, Bathymetry
    + Environment
    + Geoscientific information
    + Health
    + Imagery base maps earth cover
    + Intelligence military
    + Inland waters
    + Location
    + Oceans
    + Planning cadastre
    + Society
    + Structure
    + Transportation
    + Utilities communication
    + + + +     +     +     +
    + Keywords + + + + +
    +     +     +
    + Inspire Dataset Theme + + + + + + + + + + +
    + Services + + + + + + + + + + +
    + Related Resources + + + + + + +
    +     +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    WHERE

    +
    + GEOGRAPHIC LOCATION +
    + North Bound + + South Bound + + West Bound + + East Bound +
    + + + + + + + +
    + GEOGRAPHIC IDENTIFICATOR +
    + +
    + Reference Systems + + + + ETRS89 / Geocentric
    + ETRS89 / Geocentric 3D
    + ETRS89 / Coordenadas Geocêntricas 2D
    + ETRS89 / PT-TM06
    + Lisbon / Portuguese Grid New / Hayford-Gauss
    + Lisbon (Lisbon) / Portuguese National Grid
     + ITRF93 / Geocentric
    + ITRF93 / Geocentric 3D
    + ITRF93 / Geocentric 2D
    + ITRF93 / PTRA08 - UTM 25N
     + ITRF93 / PTRA08 - UTM 26N
     + ITRF93 / PTRA08 - UTM 28N
     + Azores Occidental 1939 / UTM zone 25N
     + Azores Central 1948 / UTM zone 26N
     + Azores Oriental 1940 / UTM zone 26N
     + Porto Santo / UTM zone 28N
     + ETRS89 / ETRS-LAEA (Lambert Azimuthal Equal Area)
    + ETRS89 / ETRS-LCC (Lambert Conformal Conic)
    + ETRS89 / ETRS-TM26N (Transverse Mercator)
    + ETRS89 / ETRS-TM27N (Transverse Mercator)
    + ETRS89 / ETRS-TM28N (Transverse Mercator)
    + ETRS89 / ETRS-TM29N (Transverse Mercator)
    + ETRS89 / ETRS-TM30N (Transverse Mercator)
    + ETRS89 / ETRS-TM31N (Transverse Mercator)
    + ETRS89 / ETRS-TM32N (Transverse Mercator)
    + ETRS89 / ETRS-TM33N (Transverse Mercator)
    + ETRS89 / ETRS-TM34N (Transverse Mercator)
    + ETRS89 / ETRS-TM35N (Transverse Mercator)
    + ETRS89 / ETRS-TM36N (Transverse Mercator)
    + ETRS89 / ETRS-TM37N (Transverse Mercator)
    + ETRS89 / ETRS-TM38N (Transverse Mercator)
    + ETRS89 / ETRS-TM39N (Transverse Mercator)
    + EVRF2000 height (European Vertical Reference Frame 2000)
    + ETRS89 + EVRF2000
    + WGS 84 / Geographic
    + Porto Santo 1995 / UTM Zona 28N
    + WGS 84 / UTM zone 28N
    + + + +     +     +
    VERTICAL EXTENT
    + Min value + + Max value + + System Identificator +
    + + + + + + EPSG:5701 - ODN height + EPSG:5780 - Cascais height + EPSG:5782 - Alicante height + EPSG:5730 - EVRF2000 + EPSG:5621 - EVRF2007 + + + + +
    + + + + + + + + + + + + + + + + + + + +
    +

    WHEN

    +
    + Temporal References + + + ( )
    +     +
    + Temporal Extension + + + / +
    +     +
    + Resource Maintenance + + + + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    HOW

    +
    + SPATIAL RESOLUTION +
    + Equivalent Scale + + + Unknown + 1: + + + Distance + + + + metre + + deg + + rad + + + + + +
    QUALITY
    + Statement + + + + +
    + Process Step
    (Description | Date | Rationale)     +     		+ + | | +
    +     +
    + Data Source + + + +
    +     +
    Conformity results
    Specification + + + +
    Specification Date + + ( ) + +
    Explanation + + + +
    + Conformance pass + + + True + False + + +
    + OTHER INFO +
    + Language + + + + +
    + Character Set + + + + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    WHO

    +
    + + + +
    + Organisation + + + + + + +
    + Individual Name + + + + +
    + Address + + , , , +
    + Telephone + + + + +
    + E-mail + + + +
    +     +
    + Online Resource + + + + + + + +
    + OTHER INFO +
    + Credits + + + +
    +     +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    DISTRIBUTION

    +
    + Online resources + + + + + + + + + + + + +
    + + + + + + + + + + + + + +
    +
    + Distribution format + + + ( version )
    +     +
    + File size + + + MB
    +     +
    + SERVICES +
    + Service type | Coupling type + + + Discovery | + View | + Download | + Transformation | + Invocation | + Other | + + | + + +
    + Operations + + DCP | Access points +
    + + + + | +
    +     +
    + LEGAL RESTRICTIONS +
    + Use limitations + + + +
    +     +
    + Access limitations + + + +
    +     +
    + Use restrictions + + + +
    +     +
    + Other restrictions + + + +
    +     +
    + Security restrictions + + + +
    +     +
    + DISTRIBUTOR +
    + Organization + + + + + + +
    + Individual name + + + + +
    + Address + + , , , +
    + Telephone + + + + +
    + E-mail + + + +
    +     +
    + Online resource + + + + + + + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    METAINFO

    +
    + Identifier + + + + +
    + Date + + + + +
    + Language + + + + +
    + Character set + + + + +
    + Metadata standard / profile + + + ( version ) + +
    + Responsible parties +
    + Organisation + + + + + + +
    + Individual Name + + + + +
    + Address + + , , , +
    + Telephone + + + + +
    + E-mail + + + +
    +     +
    + Online resource + + + + + + + +
    + +
    +
    + + + +     + +     diff --git a/geonode/catalogue/metadataxsl/tests.py b/geonode/catalogue/metadataxsl/tests.py new file mode 100644 index 0000000..bce9b26 --- /dev/null +++ b/geonode/catalogue/metadataxsl/tests.py @@ -0,0 +1,64 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.urls import reverse + +from geonode.base.models import ResourceBase +from geonode.tests.base import GeoNodeBaseTestSupport + +from geonode.base.populate_test_data import all_public, create_models, remove_models + + +class MetadataXSLTest(GeoNodeBaseTestSupport): + """ + Tests geonode.catalogue.metadataxsl app/module + """ + + type = "dataset" + + # loading test initial data + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.adm_un = "admin" + self.adm_pw = "admin" + + def test_showmetadata_access_perms(self): + _dataset = ResourceBase.objects.first() + self.assertIsNotNone(_dataset) + _dataset.set_permissions({"users": {}, "groups": {}}) + _dataset.save() + self.client.login(username=self.adm_un, password=self.adm_pw) + url = reverse("prefix_xsl_line", kwargs={"id": _dataset.id}) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.client.logout() + response = self.client.get(url) + self.assertEqual(response.status_code, 403) diff --git a/geonode/catalogue/metadataxsl/urls.py b/geonode/catalogue/metadataxsl/urls.py new file mode 100644 index 0000000..0b5e6f5 --- /dev/null +++ b/geonode/catalogue/metadataxsl/urls.py @@ -0,0 +1,23 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.urls import re_path +from . import views + +urlpatterns = [re_path(r"^xsl/(?P\d+)$", views.prefix_xsl_line, name="prefix_xsl_line")] diff --git a/geonode/catalogue/metadataxsl/views.py b/geonode/catalogue/metadataxsl/views.py new file mode 100644 index 0000000..04913c4 --- /dev/null +++ b/geonode/catalogue/metadataxsl/views.py @@ -0,0 +1,81 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import traceback +import logging + +from lxml import etree +from owslib.etree import etree as dlxml + +from django.conf import settings +from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from django.utils.translation import gettext_lazy as _ +from django.core.exceptions import PermissionDenied +from django.views.decorators.clickjacking import xframe_options_exempt + +from geonode.utils import resolve_object +from geonode.catalogue import get_catalogue +from geonode.base.models import ResourceBase + +logger = logging.getLogger(__name__) + + +@xframe_options_exempt +def prefix_xsl_line(req, id): + # if the layer is in the catalogue, try to get the distribution urls + # that cannot be precalculated. + resource = None + try: + resource = get_object_or_404(ResourceBase, pk=id) + query = {"id": resource.get_real_instance().id} + resource = resolve_object( + req, + resource.get_real_instance_class(), + query, + permission="base.view_resourcebase", + permission_msg=_("You are not permitted to view this resource"), + ) + catalogue = get_catalogue() + record = catalogue.get_record(resource.uuid) + if record: + logger.debug(record.xml) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + logger.debug(traceback.format_exc()) + msg = f'Could not connect to catalogue to save information for dataset "{str(resource)}"' + return HttpResponse(msg, status=404) + + try: + # generate an XML document (GeoNode's default is ISO) + if resource.metadata_uploaded and resource.metadata_uploaded_preserve: + md_doc = etree.tostring(dlxml.fromstring(resource.metadata_xml)) + else: + md_doc = catalogue.catalogue.csw_gen_xml(resource, settings.CATALOG_METADATA_TEMPLATE) + xml = md_doc + except Exception: + logger.debug(traceback.format_exc()) + return HttpResponse("Resource Metadata not available!") + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + xsl_static = getattr(settings, "CATALOG_METADATA_XSL", "/static/metadataxsl/metadata.xsl") + xsl_path = f"{site_url}{xsl_static}" + xsl_line = f'' + + return HttpResponse(xsl_line + xml, content_type="text/xml") diff --git a/geonode/catalogue/models.py b/geonode/catalogue/models.py new file mode 100644 index 0000000..e575be1 --- /dev/null +++ b/geonode/catalogue/models.py @@ -0,0 +1,104 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import errno +import logging + +from django.conf import settings +from django.db.models import signals +from lxml import etree +from owslib.etree import etree as dlxml +from geonode.layers.models import Dataset +from geonode.documents.models import Document +from geonode.catalogue import get_catalogue +from geonode.base.models import Link, ResourceBase + + +LOGGER = logging.getLogger(__name__) + + +def catalogue_pre_delete(instance, sender, **kwargs): + """Removes the layer from the catalogue""" + catalogue = get_catalogue() + catalogue.remove_record(instance.uuid) + + +def catalogue_post_save(instance, sender, **kwargs): + """Get information from catalogue""" + _id = instance.resourcebase_ptr.id if hasattr(instance, "resourcebase_ptr") else instance.id + resources = ResourceBase.objects.filter(id=_id) + + # Update the Catalog + try: + catalogue = get_catalogue() + catalogue.create_record(instance) + record = catalogue.get_record(instance.uuid) + except OSError as err: + msg = f'Could not connect to catalogue to save information for layer "{instance.name}"' + if err.errno == errno.ECONNREFUSED: + LOGGER.warn(msg, err) + return + else: + raise err + + if not record: + msg = f"Metadata record for {instance.title} does not exist, check the catalogue signals." + LOGGER.warning(msg) + return + + if not hasattr(record, "links"): + msg = f"Metadata record for {instance.title} should contain links." + raise Exception(msg) + + # Create the different metadata links with the available formats + if resources.exists(): + for mime, name, metadata_url in record.links["metadata"]: + try: + Link.objects.get_or_create( + resource=resources.get(), + url=metadata_url, + defaults=dict(name=name, extension="xml", mime=mime, link_type="metadata"), + ) + except Exception: + _d = dict(name=name, extension="xml", mime=mime, link_type="metadata") + Link.objects.filter( + resource=resources.get(), url=metadata_url, extension="xml", link_type="metadata" + ).update(**_d) + + if instance.metadata_uploaded and instance.metadata_uploaded_preserve: + md_doc = etree.tostring(dlxml.fromstring(instance.metadata_xml)) + else: + # generate an XML document (GeoNode's default is ISO) + raw_xml = catalogue.catalogue.csw_gen_xml(instance, settings.CATALOG_METADATA_TEMPLATE) + md_obj = dlxml.fromstring(raw_xml, parser=etree.XMLParser(remove_blank_text=True)) + md_doc = etree.tostring(md_obj, pretty_print=True, encoding="unicode") + + try: + csw_anytext = catalogue.catalogue.csw_gen_anytext(md_doc) + except Exception as e: + LOGGER.exception(e) + csw_anytext = "" + + resources.update(metadata_xml=md_doc, csw_wkt_geometry=instance.geographic_bounding_box, csw_anytext=csw_anytext) + + +if "geonode.catalogue" in settings.INSTALLED_APPS: + signals.post_save.connect(catalogue_post_save, sender=Dataset) + signals.pre_delete.connect(catalogue_pre_delete, sender=Dataset) + signals.post_save.connect(catalogue_post_save, sender=Document) + signals.pre_delete.connect(catalogue_pre_delete, sender=Document) diff --git a/geonode/catalogue/templates/catalogue/feature_catalogue.xml b/geonode/catalogue/templates/catalogue/feature_catalogue.xml new file mode 100644 index 0000000..3350b58 --- /dev/null +++ b/geonode/catalogue/templates/catalogue/feature_catalogue.xml @@ -0,0 +1,75 @@ + + + GeoNodeFeatureCatalogue + + + Generic Feature Catalogue + + + Generic Feature Catalogue + + + 1.0 + + + 2016-01-11 + + + + + {{ metadata.provider_name }} + + + {{ metadata.contact_role }} + + + + + + + {{ layer.service_typename }} + + + {{ layer.abstract }} + + + false + + + {{ layer.title }} + + + {% for attribute in attributes %} + + + + + {{ attribute.name }} + + + + + + + 1 + + + 1 + + + + + + + + + {{ attribute.type }} + + + + + + {% endfor %} + + + diff --git a/geonode/catalogue/templates/catalogue/full_metadata.xml b/geonode/catalogue/templates/catalogue/full_metadata.xml new file mode 100644 index 0000000..f5a153a --- /dev/null +++ b/geonode/catalogue/templates/catalogue/full_metadata.xml @@ -0,0 +1,518 @@ +{% load thesaurus %} +{% load l10n %} + + + {{layer.uuid}} + + + {{layer.language}} + + + utf8 + + + dataset + + {% for contact_roles, label in layer.get_defined_contact_roles %} + {% for contact_role in contact_roles %} + + + {% else %}> + {{ contact_role.first_name }} {{ contact_role.last_name}}{% endif %} + + + {% if contact_role.organization %} {{ contact_role.organization }} {% endif %} + + + {% if contact_role.position %}{{ contact_role.position }} {% endif %} + + + + + + + {% if contact_role.voice %}{{ contact_role.voice }}{% endif %} + + + {% if contact_role.fax %}{{ contact_role.fax }} {%endif %} + + + + + + + {% if contact_role.delivery %}{{ contact_role.delivery }}{% endif %} + + + {% if contact_role.city %}{{ contact_role.city }}{% endif %} + + + {% if contact_role.area %}{{ contact_role.area }}{% endif %} + + + {% if contact_role.zipcode %}{{ contact_role.zipcode }}{% endif %} + + + {% if contact_role.country %}{{ contact_role.country }}{% endif %} + + + {% if contact_role.email %}{{ contact_role.email }}{% endif %} + + + + + + + {{ SITEURL }}{{ contact_role.get_absolute_url }} + + + WWW:LINK-1.0-http--link + + + GeoNode profile page + + + + + + + {{ label }} + + + + {% endfor %} + {% endfor %} + + {{layer.csw_insert_date|date:"Y-m-d\TH:i:s\Z"}} + + + ISO 19115:2003 - Geographic information - Metadata + + + ISO 19115:2003 + + + + + + + + 4326 + + + EPSG + + + 6.11 + + + + + + + + + + + + {{layer.title}} + + {% if layer.alternate %} + + {{layer.alternate}} + + {% endif %} + + + + {{layer.date|date:"Y-m-d\TH:i:s\Z"}} + + + {{layer.date_type}} + + + + + {% if layer.edition %}{{layer.edition}}{% endif %} + + {% if layer.doi %} + + + + doi:{{ layer.doi }} + + + + {% endif %} + + mapDigital + + + + + {{layer.raw_abstract}} + + + {% if layer.raw_purpose %}{{layer.raw_purpose}}{% endif %} + + + completed + + + {% with layer.owner as owner %} + + + + {% if owner.name %} {{ owner.name }} {% endif %} + + + {% if owner.organization %} {{ owner.organization }} {% endif %} + + + {% if owner.position %}{{ owner.position }} {% endif %} + + + + + + + {% if owner.voice %}{{ owner.voice }}{% endif %} + + + {% if owner.fax %}{{ owner.fax }} {%endif %} + + + + + + + {% if owner.delivery %}{{ owner.delivery }}{% endif %} + + + {% if owner.city %}{{ owner.city }}{% endif %} + + + {% if owner.area %}{{ owner.area }}{% endif %} + + + {% if owner.zipcode %}{{ owner.zipcode }}{% endif %} + + + {% if owner.country %}{{ owner.country }}{% endif %} + + + {% if owner.email %}{{ owner.email }}{% endif %} + + + + {% if owner.user %} + + + + {{ SITEURL }}{{ layer.owner.get_absolute_url }} + + + WWW:LINK-1.0-http--link + + + GeoNode profile page + + + + {% endif %} + + + + owner + + + + {% endwith %} + + {% if layer.maintenance_frequency %} + + + + {{layer.maintenance_frequency}} + + + + {% endif %} + + + + {{ layer.get_thumbnail_url }} + + + Thumbnail for '{{layer.title}}' + + + image/png + + + + + + + {% if layer.store_type == 'raster' %} + GeoTIFF + {% else %} + ESRI Shapefile + {% endif %} + + + 1.0 + + + + {% if layer.keyword_list %} + + + {% for kw in layer.keyword_list %} + + {{kw}} + + {% endfor %} + + theme + + + + {% endif %} + {% if layer.tkeywords %} + {% for thesaurus_id in layer.tkeywords|get_unique_thesaurus_set %} + + + {% for keyword in layer.tkeywords.all %} + {% if keyword.thesaurus.id == thesaurus_id %} + + {{keyword.alt_label}} + + {% endif %} + {% endfor %} + + + + {{thesaurus_id|get_thesaurus_title}} + + + + + {{thesaurus_id|get_thesaurus_date}} + + + pubblicazione + + + + + + + + {% endfor %} + {% endif %} + {% for region in layer.regions.all %} + + + + {{region.name}} + + + place + + + + {% endfor %} + {% if LICENSES_METADATA == 'light' and layer.license %} + + + + license + + + {{layer.license_light}} + + {% if layer.doi %} + + DOI: {{layer.doi}} + + {% endif %} + + + {% endif %} + {% if LICENSES_METADATA == 'verbose' and layer.license %} + + + + license + + + {{layer.license_verbose}} + + + + {% endif %} + + + + {{layer.restriction_code_type.identifier}} + + + {{layer.raw_constraints_other}} + + + + {% if layer.spatial_representation_type %} + + {{layer.spatial_representation_type.identifier}} + + {% endif %} + + {{layer.language}} + + + utf8 + + + {% if layer.category %} + {{layer.category.identifier}} + {% endif %} + + + + + + {% localize off %} + + {{layer.ll_bbox.0}} + + + {{layer.ll_bbox.1}} + + + {{layer.ll_bbox.2}} + + + {{layer.ll_bbox.3}} + + {% endlocalize %} + + + + + {% if layer.temporal_extent_start and layer.temporal_extent_end %} + + + + + + + {{layer.temporal_extent_start|date:"c"}} + {{layer.temporal_extent_end|date:"c"}} + + + + + + + {% endif %} + + {{ layer.raw_supplemental_information }} + + + + + + {% if layer.subtype == 'raster' %} + + + + image + + + {% elif layer.subtype == 'vector' %} + + + 0 + + + + {% endif %} + + + + + + + + + {{ SITEURL }}{% url 'resolve_uuid' uuid=layer.uuid %} + + + WWW:LINK-1.0-http--link + + + Online link to the '{{layer.title}}' description on GeoNode + + + + {% for link in layer.link_set.download %} + + + + {{link.url}} + + + WWW:DOWNLOAD-1.0-http--download + + + {{layer.name}}.{{link.extension}} + + + {{layer.title}} ({{link.name}} Format) + + + + {% endfor %} + {% for link in layer.link_set.ows %} + + + + {{link.url}} + + + {{link.link_type}} + + + {{ layer.alternate }} + + + {{layer.workspace}} Service - Provides Layer: {{ layer.title }} + + + + {% endfor %} + + + + + + + + + + dataset + + + + + + {% else %}>{{layer.raw_data_quality_statement}}{% endif %} + + + + + + \ No newline at end of file diff --git a/geonode/catalogue/templates/catalogue/opensearch_description.xml b/geonode/catalogue/templates/catalogue/opensearch_description.xml new file mode 100644 index 0000000..96d8d20 --- /dev/null +++ b/geonode/catalogue/templates/catalogue/opensearch_description.xml @@ -0,0 +1,14 @@ + + + {{ shortname }} + {{ shortname }} + Search {{ url }} + {{ tags }} + {{ url }}/static/geonode/img/favicon.ico + UTF-8 + + + {{ developer }} + {{ contact }} + {{ attribution }} + diff --git a/geonode/catalogue/templates/catalogue/transaction_delete.xml b/geonode/catalogue/templates/catalogue/transaction_delete.xml new file mode 100644 index 0000000..0de44e1 --- /dev/null +++ b/geonode/catalogue/templates/catalogue/transaction_delete.xml @@ -0,0 +1,13 @@ + + + + + + + {{ id_pname }} + {{ layer.uuid }} + + + + + diff --git a/geonode/catalogue/templates/catalogue/transaction_insert.xml b/geonode/catalogue/templates/catalogue/transaction_insert.xml new file mode 100644 index 0000000..23f79f8 --- /dev/null +++ b/geonode/catalogue/templates/catalogue/transaction_insert.xml @@ -0,0 +1,6 @@ + + + + {% include CATALOG_METADATA_TEMPLATE %} + + diff --git a/geonode/catalogue/templates/catalogue/transaction_update.xml b/geonode/catalogue/templates/catalogue/transaction_update.xml new file mode 100644 index 0000000..0637f44 --- /dev/null +++ b/geonode/catalogue/templates/catalogue/transaction_update.xml @@ -0,0 +1,6 @@ + + + + {% include CATALOG_METADATA_TEMPLATE %} + + diff --git a/geonode/catalogue/templates/geonode_metadata_full.html b/geonode/catalogue/templates/geonode_metadata_full.html new file mode 100644 index 0000000..8e46954 --- /dev/null +++ b/geonode/catalogue/templates/geonode_metadata_full.html @@ -0,0 +1,180 @@ +{% load i18n %} +{% load static %} + + + + + + + {{ resource.title }} + {% block head %} + {% if DEBUG_STATIC %} + + + + + + {% else %} + + {% endif %} + + + + + {% endblock %} + + + {% block body_outer %} + {% block body %}{% endblock body %} + {% block sidebar %}{% endblock sidebar %} + {% endblock body_outer %} +
    +
    +
    +
    +
    +

    {% trans "Resource Metadata" %}

    +

    {{ resource.title }}

    +
    +
    +
    + +
    +
    +
    +
    {% trans "Thumbnail" %}
    +
    {{ resource.title }}
    + +
    {% trans "Resource ID" %}
    +
    {{resource.uuid}}
    + +
    {% trans "Title" %}
    +
    {{resource.title}}
    + +
    {% trans "Date" %}
    +
    {{resource.date}}, {% trans resource.date_type|title %}
    + +
    {% trans "Abstract" %}
    +
    {{resource.raw_abstract}}
    + +
    {% trans "Edition" %}
    + {% if resource.edition %} +
    {{resource.edition}}
    + {% else %} +
    --
    + {% endif %} + + +
    {% trans "Responsible" %}
    +
    {{resource.owner}}
    + + {% for role in extra_res_md.roles %} +
    {% trans role.label %}
    + + {% for user in role.users %} +
    {{user.last_name}}
    +
    {{ user.email}}
    + {% endfor %} + {% endfor %} + +
    {% trans "Purpose" %}
    + {% if resource.purpose %} +
    {{resource.raw_purpose}}
    + {% else %} +
    --
    + {% endif %} + + +
    {% trans "Maintenance Frequency" %}
    +
    {{resource.maintenance_frequency}}
    + +
    {% trans "Type" %}
    +
    {{extra_res_md.sprt_identifier}}
    + +
    {% trans "Restrictions" %}
    +
    {{resource.restriction_code_type}}
    +
    {{resource.raw_constraints_other}}
    + +
    {% trans "License" %}
    +
    {{resource.license}}
    + +
    {% trans "Language" %}
    +
    {{resource.language}}
    + +
    {% trans "Temporal Extent" %}
    +
    {% trans "Start" %}
    + {% if resource.temporal_extent_start %} +
    {{resource.temporal_extent_start}}
    + {% else %} +
    --
    + {% endif %} +
    {% trans "End" %}
    + {% if resource.temporal_extent_end %} +
    {{resource.temporal_extent_end}}
    + {% else %} +
    --
    + {% endif %} + + +
    {% trans "Supplemental Information" %}
    +
    {{resource.raw_supplemental_information}}
    + +
    {% trans "Data Quality" %}
    + {% if resource.data_quality_statement %} +
    {{resource.raw_data_quality_statement}}
    + {% else %} +
    --
    + {% endif %} + +
    {% trans "Extent" %}
    +
    +
      +
    • x0: {{resource.bbox_helper.xmin}}
      • +
    • x1: {{resource.bbox_helper.xmax}}
      • +
    • y0: {{resource.bbox_helper.ymin}}
      • +
    • y1: {{resource.bbox_helper.ymax}}
      • +
    +
    + +
    {% trans "Spatial Reference System Identifier" %}
    +
    {{resource.srid}}
    + +
    {% trans "Keywords" %}
    +
    {{extra_res_md.keywords}}
    + +
    {% trans "Category" %}
    +
    {{resource.category}}
    + + {% if resource.regions.all %} +
    {% trans "Regions" %}
    +
    + {% for region in resource.regions.all %} + {{ region.name }} + {% if not forloop.last %},{% endif %} + {% endfor %} +
    + {% endif %} + + {% if "dataset" in request.META.HTTP_REFERER|stringformat:"s" %} +
    {% trans "Attributes" %}
    +
    + +
    +
    + + {% endif %} +
    +
    +
    +
    + + diff --git a/geonode/catalogue/templates/geonode_metadata_full.txt b/geonode/catalogue/templates/geonode_metadata_full.txt new file mode 100644 index 0000000..e69de29 diff --git a/geonode/catalogue/tests.py b/geonode/catalogue/tests.py new file mode 100644 index 0000000..4998881 --- /dev/null +++ b/geonode/catalogue/tests.py @@ -0,0 +1,180 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +import xml.etree.ElementTree as ET + +from django.db.models import Q +from django.test import RequestFactory +from django.http.response import Http404 +from django.core.exceptions import PermissionDenied +from geonode.layers.models import Dataset +from geonode.catalogue import get_catalogue +from django.contrib.auth import get_user_model +from django.contrib.auth.models import AnonymousUser +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.catalogue.models import catalogue_post_save + +from geonode.catalogue.views import csw_global_dispatch, resolve_uuid +from geonode.layers.populate_datasets_data import create_dataset_data + +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset + +logger = logging.getLogger(__name__) + + +class CatalogueTest(GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.request = self.__request_factory_single(123) + create_dataset_data() + self.user = "admin" + self.passwd = "admin" + + def test_get_catalog(self): + """Tests the get_catalogue function works.""" + c = get_catalogue() + self.assertIsNotNone(c) + + def test_update_metadata_records(self): + dataset = Dataset.objects.first() + self.assertIsNotNone(dataset) + dataset.abstract = "

    Test HTML abstract

    " + dataset.save() + self.assertEqual(dataset.abstract, "

    Test HTML abstract

    ") + self.assertEqual(dataset.raw_abstract, "Test HTML abstract") + # refresh catalogue metadata records + catalogue_post_save(instance=dataset, sender=dataset.__class__) + # get all records + csw = get_catalogue() + record = csw.get_record(dataset.uuid) + self.assertIsNotNone(record) + self.assertEqual(record.identification[0].title, dataset.title) + self.assertEqual(record.identification[0].abstract, dataset.raw_abstract) + if len(record.identification[0].otherconstraints) > 0: + self.assertEqual(record.identification[0].otherconstraints[0], dataset.raw_constraints_other) + + def test_given_a_simple_request_should_return_200(self): + actual = csw_global_dispatch(self.request) + self.assertEqual(200, actual.status_code) + + def test_given_a_request_for_a_single_dataset_should_return_single_value_in_xml_without_dataset_filter(self): + dataset = Dataset.objects.first() + request = self.__request_factory_single(dataset.uuid) + response = csw_global_dispatch(request) + root = ET.fromstring(response.content) + actual = len(list(root)) + self.assertEqual(1, actual) + + def test_given_a_request_for_a_single_dataset_should_return_empty_value_in_xml_with_dataset_filter(self): + dataset = Dataset.objects.first() + request = self.__request_factory_single(dataset.uuid) + response = csw_global_dispatch(request, self.dataset_filter) + root = ET.fromstring(response.content) + actual = len(list(root)) + self.assertEqual(0, actual) + + def test_given_a_request_for_multiple_dataset_should_return_empty_value_in_xml_with_dataset_filter(self): + request = self.__request_factory_multiple() + response = csw_global_dispatch(request, self.dataset_filter) + root = ET.fromstring(response.content) + actual = root.find("{http://www.opengis.net/cat/csw/2.0.2}SearchResults").attrib["numberOfRecordsReturned"] + self.assertEqual(0, int(actual)) + + def test_given_a_request_for_multiple_dataset_should_return_multiple_value_in_xml_with_dataset_filter(self): + request = self.__request_factory_multiple() + response = csw_global_dispatch(request, self.dataset_filter_multiple) + root = ET.fromstring(response.content) + actual = root.find("{http://www.opengis.net/cat/csw/2.0.2}SearchResults").attrib["numberOfRecordsReturned"] + self.assertEqual(2, int(actual)) + + @staticmethod + def dataset_filter(dataset): + return dataset.filter(uuid__startswith="foo_uuid") + + @staticmethod + def dataset_filter_multiple(dataset): + return dataset.filter(Q(title="CA") | Q(title="uniquetitle")) + + @staticmethod + def __request_factory_single(uuid): + factory = RequestFactory() + url = "http://localhost:8000/catalogue/csw?request=GetRecordById" + url += f"&service=CSW&version=2.0.2&id={uuid}" + url += "&outputschema=http%3A%2F%2Fwww.isotc211.org%2F2005%2Fgmd&elementsetname=full" + request = factory.get(url) + + request.user = AnonymousUser() + return request + + @staticmethod + def __request_factory_multiple(): + factory = RequestFactory() + url = "http://localhost:8000/catalogue/csw/?request=GetRecords&service=CSW&version=2.0.2" + url += "&outputschema=http%3A%2F%2Fwww.isotc211.org%2F2005%2Fgmd&elementsetname=full" + url += "&typenames=gmd:MD_Metadata" + request = factory.get(url) + request.user = get_user_model().objects.first() + return request + + +class UUIDResolverTest(GeoNodeBaseTestSupport): + def setUp(self): + self.dataset = create_single_dataset(name="test_uuid_resolver_dataset") + + def tearDown(self): + Dataset.objects.filter(name="test_uuid_resolver_dataset").delete() + + def test_uuid_resolver_existing_dataset(self): + user = get_user_model().objects.first() + self.dataset.set_default_permissions(owner=user) + request = RequestFactory().get(f"http://localhost:8000/catalogue/uuid/{self.dataset.uuid}") + request.user = user + response = resolve_uuid(request, self.dataset.uuid) + self.assertEqual(302, response.status_code) + self.assertEqual(f"/catalogue/#/dataset/{self.dataset.pk}", response.headers["Location"]) + + def test_uuid_resolver_non_existing_dataset(self): + user = get_user_model().objects.first() + self.dataset.set_default_permissions(owner=user) + request = RequestFactory().get("http://localhost:8000/catalogue/uuid/asdfasdf") + request.user = user + with self.assertRaises(Http404) as context: + resolve_uuid(request, "asdfasdf") + self.assertTrue("No ResourceBase matches the given query." in str(context.exception)) + + def test_uuid_resolver_missing_permissions(self): + self.dataset.set_permissions( + {"groups": {"registered-members": ["base.view_resourcebase", "base.download_resourcebase"]}} + ) + request = RequestFactory().get(f"http://localhost:8000/catalogue/uuid/{self.dataset.uuid}") + request.user = AnonymousUser() + with self.assertRaises(PermissionDenied) as context: + resolve_uuid(request, self.dataset.uuid) + self.assertTrue("Permission Denied" in str(context.exception)) diff --git a/geonode/catalogue/urls.py b/geonode/catalogue/urls.py new file mode 100644 index 0000000..e5816e3 --- /dev/null +++ b/geonode/catalogue/urls.py @@ -0,0 +1,38 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.urls import re_path +from django.urls import path +from . import views + +urlpatterns = [ + re_path(r"^csw$", views.csw_global_dispatch, name="csw_global_dispatch"), + re_path(r"^opensearch$", views.opensearch_dispatch, name="opensearch_dispatch"), + re_path( + r"^csw_to_extra_format/(?P[^/]*)/(?P[^/]*).txt$", + views.csw_render_extra_format_txt, + name="csw_render_extra_format_txt", + ), + re_path( + r"^csw_to_extra_format/(?P[^/]*)/(?P[^/]*).html$", + views.csw_render_extra_format_html, + name="csw_render_extra_format_html", + ), + path(r"uuid/", views.resolve_uuid, name="resolve_uuid"), +] diff --git a/geonode/catalogue/views.py b/geonode/catalogue/views.py new file mode 100644 index 0000000..8e64559 --- /dev/null +++ b/geonode/catalogue/views.py @@ -0,0 +1,344 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import logging +from django.conf import settings +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render, redirect +from django.contrib.auth import get_user_model +from django.views.decorators.csrf import csrf_exempt +from pycsw import server +from guardian.shortcuts import get_objects_for_user +from geonode.catalogue.backends.pycsw_local import CONFIGURATION +from geonode.base.models import ResourceBase +from geonode.layers.models import Dataset +from geonode.base.auth import get_or_create_token +from geonode.base.models import SpatialRepresentationType +from geonode.groups.models import GroupProfile +from geonode.utils import resolve_object +from django.db import connection +from django.core.exceptions import ObjectDoesNotExist +from geonode.people import Roles + + +@csrf_exempt +def csw_global_dispatch(request, dataset_filter=None, config_updater=None): + """pycsw wrapper""" + + # this view should only operate if pycsw_local is the backend + # else, redirect to the URL of the non-pycsw_local backend + if settings.CATALOGUE["default"]["ENGINE"] != "geonode.catalogue.backends.pycsw_local": + return HttpResponseRedirect(settings.CATALOGUE["default"]["URL"]) + + mdict = dict(settings.PYCSW["CONFIGURATION"], **CONFIGURATION) + mdict = config_updater(mdict) if config_updater else mdict + + access_token = None + if request and request.user: + access_token = get_or_create_token(request.user) + if access_token and access_token.is_expired(): + access_token = None + + absolute_uri = f"{request.build_absolute_uri()}" + query_string = f"{request.META['QUERY_STRING']}" + env = request.META.copy() + + if access_token and not access_token.is_expired(): + env.update({"access_token": access_token.token}) + if "access_token" not in query_string: + absolute_uri = f"{absolute_uri}&access_token={access_token.token}" + query_string = f"{query_string}&access_token={access_token.token}" + + env.update({"local.app_root": os.path.dirname(__file__), "REQUEST_URI": absolute_uri, "QUERY_STRING": query_string}) + + # Save original filter before doing anything + mdict_filter = mdict["repository"]["filter"] + + try: + # Filter out Layers not accessible to the User + authorized_ids = [] + if request.user: + profiles = get_user_model().objects.filter(username=str(request.user)) + else: + profiles = get_user_model().objects.filter(username="AnonymousUser") + if profiles: + authorized = list(get_objects_for_user(profiles[0], "base.view_resourcebase").values("id")) + + layers = ResourceBase.objects.filter(id__in=[d["id"] for d in authorized]) + + if dataset_filter and layers: + layers = dataset_filter(layers) + + if layers: + authorized_ids = [d.id for d in layers] + + if len(authorized_ids) > 0: + authorized_datasets = f"({', '.join(str(e) for e in authorized_ids)})" + authorized_datasets_filter = f"id IN {authorized_datasets}" + mdict["repository"]["filter"] += f" AND {authorized_datasets_filter}" + if request.user and request.user.is_authenticated: + mdict["repository"]["filter"] = f"({mdict['repository']['filter']}) OR ({authorized_datasets_filter})" + else: + authorized_datasets_filter = "id = -9999" + mdict["repository"]["filter"] += f" AND {authorized_datasets_filter}" + + # Filter out Documents and Maps + if "ALTERNATES_ONLY" in settings.CATALOGUE["default"] and settings.CATALOGUE["default"]["ALTERNATES_ONLY"]: + mdict["repository"]["filter"] += " AND alternate IS NOT NULL" + + # Filter out Layers belonging to specific Groups + is_admin = False + if request.user: + is_admin = request.user.is_superuser if request.user else False + + if not is_admin and settings.GROUP_PRIVATE_RESOURCES: + groups_ids = [] + if request.user and request.user.is_authenticated: + for group in request.user.groups.all(): + groups_ids.append(group.id) + group_list_all = [] + try: + group_list_all = request.user.group_list_all().values("group") + except Exception: + pass + for group in group_list_all: + if isinstance(group, dict): + if "group" in group: + groups_ids.append(group["group"]) + else: + groups_ids.append(group.id) + + public_groups = GroupProfile.objects.exclude(access="private").values("group") + for group in public_groups: + if isinstance(group, dict): + if "group" in group: + groups_ids.append(group["group"]) + else: + groups_ids.append(group.id) + + if len(groups_ids) > 0: + groups = f"({', '.join(str(e) for e in groups_ids)})" + groups_filter = f"(group_id IS NULL OR group_id IN {groups})" + mdict["repository"]["filter"] += f" AND {groups_filter}" + else: + groups_filter = "group_id IS NULL" + mdict["repository"]["filter"] += f" AND {groups_filter}" + + csw = server.Csw(mdict, env, version="2.0.2") + + content = csw.dispatch_wsgi() + + # pycsw 2.0 has an API break: + # - pycsw < 2.0: content = xml_response + # - pycsw >= 2.0: content = [http_status_code, content] + # deal with the API break + + if isinstance(content, list): # pycsw 2.0+ + content = content[1] + + finally: + # Restore original filter before doing anything + mdict["repository"]["filter"] = mdict_filter + + return HttpResponse(content, content_type=csw.contenttype) + + +@csrf_exempt +def opensearch_dispatch(request): + """OpenSearch wrapper""" + + ctx = { + "shortname": settings.PYCSW["CONFIGURATION"]["metadata:main"]["identification_title"], + "description": settings.PYCSW["CONFIGURATION"]["metadata:main"]["identification_abstract"], + "developer": settings.PYCSW["CONFIGURATION"]["metadata:main"]["contact_name"], + "contact": settings.PYCSW["CONFIGURATION"]["metadata:main"]["contact_email"], + "attribution": settings.PYCSW["CONFIGURATION"]["metadata:main"]["provider_name"], + "tags": settings.PYCSW["CONFIGURATION"]["metadata:main"]["identification_keywords"].replace(",", " "), + "url": settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL, + } + + return render( + request, + "catalogue/opensearch_description.xml", + context=ctx, + content_type="application/opensearchdescription+xml", + ) + + +# transforms a row sql query into a two dimension array +def dictfetchall(cursor): + """Generate all rows from a cursor as a dict""" + for row in cursor.fetchall(): + yield {col[0]: row for col in cursor.description} + + +# choose separators +def get_CSV_spec_char(): + return {"separator": ";", "carriage_return": "\r\n"} + + +# format value to unicode str without ';' char +def fst(value): + chrs = get_CSV_spec_char() + result = str(value) + result = result.replace(chrs["separator"], ",").replace("\\n", " ").replace("\r\n", " ") + return result + + +# from a resource object, build the corresponding metadata dict +# the aim is to handle the output format (csv, html or pdf) the same structure +def build_md_dict(resource): + md_dict = { + "r_uuid": {"label": "uuid", "value": resource.uuid}, + "r_title": {"label": "titre", "value": resource.title}, + } + return md_dict + + +def get_keywords(resource): + content = " " + cursor = connection.cursor() + cursor.execute( + "SELECT a.*,b.* FROM taggit_taggeditem as a,taggit_tag" " as b WHERE a.object_id = %s AND a.tag_id=b.id", + [resource.id], + ) + for x in dictfetchall(cursor): + content += f"{fst(x['name'])}, " + return content[:-2] + + +# from a resource uuid, return a httpResponse +# containing the whole geonode metadata +@csrf_exempt +def csw_render_extra_format_txt(request, layeruuid, resname): + """pycsw wrapper""" + resource = ResourceBase.objects.get(uuid=layeruuid) + chrs = get_CSV_spec_char() + s = chrs["separator"] + c = chrs["carriage_return"] + sc = s + c + content = f"Resource metadata{sc}" + content += f"uuid{s}{fst(resource.uuid)}{sc}" + content += f"title{s}{fst(resource.title)}{sc}" + content += f"resource owner{s}{fst(resource.owner)}{sc}" + content += f"date{s}{fst(resource.date)}{sc}" + content += f"date type{s}{fst(resource.date_type)}{sc}" + content += f"abstract{s}{fst(resource.abstract)}{sc}" + content += f"edition{s}{fst(resource.edition)}{sc}" + content += f"purpose{s}{fst(resource.purpose)}{sc}" + content += f"maintenance frequency{s}{fst(resource.maintenance_frequency)}{sc}" + + try: + sprt = SpatialRepresentationType.objects.get(id=resource.spatial_representation_type_id) + content += f"identifier{s}{fst(sprt.identifier)}{sc}" + except ObjectDoesNotExist: + content += f"ObjectDoesNotExist{sc}" + + content += f"restriction code type{s}{fst(resource.restriction_code_type)}{sc}" + content += f"constraints other {s}{fst(resource.constraints_other)}{sc}" + content += f"license{s}{fst(resource.license)}{sc}" + content += f"language{s}{fst(resource.language)}{sc}" + content += f"temporal extent{sc}" + content += f"temporal extent start{s}{fst(resource.temporal_extent_start)}{sc}" + content += f"temporal extent end{s}{fst(resource.temporal_extent_end)}{sc}" + content += f"supplemental information{s}{fst(resource.supplemental_information)}{sc}" + """content += 'URL de distribution ' + s + fst( + resource.distribution_url) + sc""" + """content += 'description de la distribution' + s + fst( + resource.distribution_description) + sc""" + content += f"data quality statement{s}{fst(resource.data_quality_statement)}{sc}" + + ext = resource.bbox_polygon.extent + content += f"extent {s}{fst(ext[0])},{fst(ext[2])},{fst(ext[1])},{fst(ext[3])}{sc}" + content += f"SRID {s}{fst(resource.srid)}{sc}" + content += f"Thumbnail url{s}{fst(resource.thumbnail_url)}{sc}" + + content += f"keywords;{get_keywords(resource)}{s}" + content += f"category{s}{fst(resource.category)}{sc}" + + content += f"regions{s}" + for reg in resource.regions.all(): + content += f"{fst(reg.name_en)}," + content = content[:-1] + content += sc + + if resource.detail_url.find("/datasets/") > -1: + layer = Dataset.objects.get(resourcebase_ptr_id=resource.id) + content += f"attribute data{sc}" + content += "attribute name;label;description\n" + for attr in layer.attribute_set.all(): + content += fst(attr.attribute) + s + content += fst(attr.attribute_label) + s + content += fst(attr.description) + sc + + @staticmethod + def __append_contact_role__(content, cr_attr_name, title_in_txt): + cr = resource.__getattribute__(cr_attr_name) + if cr is not None or (isinstance(list, cr) and len(0)): + content += f"{title_in_txt}{sc}" + for user in cr: + content += f"name{s}{fst(user.last_name)}{sc}" + content += f"e-mail{s}{fst(user.email)}{sc}" + return content + + for role in set(Roles).difference([Roles.OWNER]): + content = __append_contact_role__(content, role.name, role.label) + + logger = logging.getLogger(__name__) + logger.error(content) + + return HttpResponse(content, content_type="text/csv") + + +def csw_render_extra_format_html(request, layeruuid, resname): + resource = ResourceBase.objects.get(uuid=layeruuid) + extra_res_md = {} + try: + sprt = SpatialRepresentationType.objects.get(id=resource.spatial_representation_type_id) + extra_res_md["sprt_identifier"] = sprt.identifier + except ObjectDoesNotExist: + extra_res_md["sprt_identifier"] = "not filled" + kw = get_keywords(resource) + if len(kw) == 0: + extra_res_md["keywords"] = "no keywords" + else: + extra_res_md["keywords"] = get_keywords(resource) + + if resource.detail_url.find("/datasets/") > -1: + layer = Dataset.objects.get(resourcebase_ptr_id=resource.id) + extra_res_md["atrributes"] = "" + for attr in layer.attribute_set.all(): + s = f"{attr.attribute}{attr.attribute_label}{attr.description}" + extra_res_md["atrributes"] += s + + extra_res_md["roles"] = [] + for role in Roles: + cr = resource.__getattribute__(role.name) + if not isinstance(cr, list): + cr = [cr] + users = [{"pk": user.id, "last_name": user.last_name, "email": user.email} for user in cr] + if users: + extra_res_md["roles"].append({"label": role.label, "users": users}) + + return render(request, "geonode_metadata_full.html", context={"resource": resource, "extra_res_md": extra_res_md}) + + +def resolve_uuid(request, uuid): + resource = resolve_object(request, ResourceBase, {"uuid": uuid}) + return redirect(resource) diff --git a/geonode/celery_app.py b/geonode/celery_app.py new file mode 100644 index 0000000..5f9b5e9 --- /dev/null +++ b/geonode/celery_app.py @@ -0,0 +1,63 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import logging + +from celery import Celery + +# from celery.schedules import crontab + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geonode.settings") +app = Celery("geonode") +logger = logging.getLogger(__name__) + + +def _log(msg, *args): + logger.debug(msg, *args) + + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() +app.autodiscover_tasks(packages=["geonode.harvesting.harvesters"]) + +""" CELERAY SAMPLE TASKS +@app.on_after_configure.connect +def setup_periodic_tasks(sender, **kwargs): + # Calls test('hello') every 10 seconds. + sender.add_periodic_task(10.0, test.s('hello'), name='add every 10') + + # Calls test('world') every 30 seconds + sender.add_periodic_task(30.0, test.s('world'), expires=10) + +@app.task( + bind=True, + name='geonode.test', + queue='default') +def test(arg): + _log(arg) + +@app.task( + bind=True, + name='geonode.debug_task', + queue='default') +def debug_task(self): + _log(f"Request: {self.request}") +""" diff --git a/geonode/client/__init__.py b/geonode/client/__init__.py new file mode 100644 index 0000000..204aed5 --- /dev/null +++ b/geonode/client/__init__.py @@ -0,0 +1,22 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from pkgutil import extend_path + + +__path__ = extend_path(__path__, __name__) # noqa diff --git a/geonode/client/apps.py b/geonode/client/apps.py new file mode 100644 index 0000000..f24a330 --- /dev/null +++ b/geonode/client/apps.py @@ -0,0 +1,26 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig as BaseAppConfig +from django.utils.translation import gettext_lazy as _ + + +class AppConfig(BaseAppConfig): + name = "geonode.client" + label = "geonode_client" + verbose_name = _("GeoNode Client Library") diff --git a/geonode/client/conf.py b/geonode/client/conf.py new file mode 100644 index 0000000..dfdd6c4 --- /dev/null +++ b/geonode/client/conf.py @@ -0,0 +1,57 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import importlib + +from django.conf import settings # noqa +from django.core.exceptions import ImproperlyConfigured + +from appconf import AppConf + + +def load_path_attr(path): + i = path.rfind(".") + module, attr = path[:i], path[i + 1 :] + try: + mod = importlib.import_module(module) + except ImportError as e: + raise ImproperlyConfigured(f"Error importing {module}: '{e}'") + try: + attr = getattr(mod, attr) + except AttributeError: + raise ImproperlyConfigured(f"Module '{module}' does not define a '{attr}'") + return attr + + +def is_installed(package): + try: + __import__(package) + return True + except ImportError: + return False + + +class GeoNodeClientAppConf(AppConf): + LAYER_PREVIEW_LIBRARY = "geonode" + HOOKSET = "geonode.client.hooksets.BaseHookSet" + + def configure_hookset(self, value): + return load_path_attr(value)() + + class Meta: + prefix = "geonode_client" diff --git a/geonode/client/hooks.py b/geonode/client/hooks.py new file mode 100644 index 0000000..686bc14 --- /dev/null +++ b/geonode/client/hooks.py @@ -0,0 +1,36 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from .conf import settings + + +class HookProxy: + def __getattr__(self, attr): + if not isinstance(settings.GEONODE_CLIENT_HOOKSET, str): + return getattr(settings.GEONODE_CLIENT_HOOKSET, attr) + else: + import importlib + + cls = settings.GEONODE_CLIENT_HOOKSET.split(".") + module_name, class_name = (".".join(cls[:-1]), cls[-1]) + i = importlib.import_module(module_name) + hook = getattr(i, class_name)() + return getattr(hook, attr) + + +hookset = HookProxy() diff --git a/geonode/client/hooksets.py b/geonode/client/hooksets.py new file mode 100644 index 0000000..4745a46 --- /dev/null +++ b/geonode/client/hooksets.py @@ -0,0 +1,147 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import json + + +class BaseHookSet: + # Layers + def dataset_list_template(self, context=None): + return NotImplemented + + def dataset_detail_template(self, context=None): + return NotImplemented + + def dataset_new_template(self, context=None): + return NotImplemented + + def dataset_view_template(self, context=None): + return NotImplemented + + def dataset_edit_template(self, context=None): + return NotImplemented + + def dataset_update_template(self, context=None): + return NotImplemented + + def dataset_embed_template(self, context=None): + return NotImplemented + + def dataset_download_template(self, context=None): + return NotImplemented + + def dataset_style_edit_template(self, context=None): + return NotImplemented + + def dataset_list_url(self): + return NotImplemented + + def dataset_upload_url(self): + return NotImplemented + + def dataset_detail_url(self, layer): + return NotImplemented + + # Maps + def map_list_template(self, context=None): + return NotImplemented + + def map_detail_template(self, context=None): + return NotImplemented + + def map_new_template(self, context=None): + return NotImplemented + + def map_view_template(self, context=None): + return NotImplemented + + def map_edit_template(self, context=None): + return NotImplemented + + def map_update_template(self, context=None): + return NotImplemented + + def map_embed_template(self, context=None): + return NotImplemented + + def map_download_template(self, context=None): + return NotImplemented + + def map_list_url(self): + return NotImplemented + + def map_detail_url(self, map): + return NotImplemented + + # GeoApps + def geoapp_list_template(self, context=None): + return NotImplemented + + def geoapp_detail_template(self, context=None): + return NotImplemented + + def geoapp_new_template(self, context=None): + return NotImplemented + + def geoapp_view_template(self, context=None): + return NotImplemented + + def geoapp_edit_template(self, context=None): + return NotImplemented + + def geoapp_update_template(self, context=None): + return NotImplemented + + def geoapp_embed_template(self, context=None): + return NotImplemented + + def geoapp_download_template(self, context=None): + return NotImplemented + + def geoapp_list_url(self): + return NotImplemented + + def geoapp_detail_url(self, geoapp): + return NotImplemented + + def resourcebase_embed_template(self, context=None): + return None + + # Documents + def document_list_url(self): + return NotImplemented + + def document_detail_url(self, document): + return NotImplemented + + # Map Persisting + def viewer_json(self, conf, context=None): + if isinstance(conf, str): + conf = json.loads(conf) + return conf + + def update_from_viewer(self, conf, context=None): + return NotImplemented + + def metadata_update_redirect(self, url, request=None): + if "metadata_uri" in url: + return url.replace("/metadata_uri", "") + return url.replace("/metadata", "") + + def get_absolute_url(self, context=None): + return None diff --git a/geonode/client/migrations/0001_initial.py b/geonode/client/migrations/0001_initial.py new file mode 100644 index 0000000..a04cf6d --- /dev/null +++ b/geonode/client/migrations/0001_initial.py @@ -0,0 +1,44 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='GeoNodeThemeCustomization', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('identifier', models.CharField(max_length=255, editable=False)), + ('name', models.CharField(max_length=100)), + ('date', models.DateTimeField(auto_now_add=True)), + ('description', models.TextField(null=True, blank=True)), + ('is_enabled', models.BooleanField(default=False)), + ('logo', models.ImageField(null=True, upload_to='img/%Y/%m', blank=True)), + ('jumbotron_bg', models.ImageField(null=True, upload_to='img/%Y/%m', blank=True)), + ('body_color', models.CharField(default='#333333', max_length=10)), + ('navbar_color', models.CharField(default='#333333', max_length=10)), + ('jumbotron_color', models.CharField(default='#2c689c', max_length=10)), + ('copyright_color', models.CharField(default='#2c689c', max_length=10)), + ('contactus', models.BooleanField(default=False)), + ('copyright', models.TextField(null=True, blank=True)), + ('contact_name', models.TextField(null=True, blank=True)), + ('contact_position', models.TextField(null=True, blank=True)), + ('contact_administrative_area', models.TextField(null=True, blank=True)), + ('contact_street', models.TextField(null=True, blank=True)), + ('contact_postal_code', models.TextField(null=True, blank=True)), + ('contact_city', models.TextField(null=True, blank=True)), + ('contact_country', models.TextField(null=True, blank=True)), + ('contact_delivery_point', models.TextField(null=True, blank=True)), + ('contact_voice', models.TextField(null=True, blank=True)), + ('contact_facsimile', models.TextField(null=True, blank=True)), + ('contact_email', models.TextField(null=True, blank=True)), + ], + options={ + 'ordering': ('date',), + 'verbose_name_plural': 'Themes', + }, + ), + ] diff --git a/geonode/client/migrations/0002_auto_20180412_1039.py b/geonode/client/migrations/0002_auto_20180412_1039.py new file mode 100644 index 0000000..64a6ca5 --- /dev/null +++ b/geonode/client/migrations/0002_auto_20180412_1039.py @@ -0,0 +1,35 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geonode_client', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Partner', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('logo', models.ImageField(null=True, upload_to='img/%Y/%m', blank=True)), + ('name', models.CharField(max_length=100)), + ('title', models.CharField(max_length=255)), + ('href', models.CharField(max_length=255)), + ], + options={ + 'ordering': ('name',), + 'verbose_name_plural': 'Partners', + }, + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='partners_title', + field=models.CharField(default='Our Partners', max_length=100, null=True, blank=True), + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='partners', + field=models.ManyToManyField(related_name='partners', to='geonode_client.Partner', blank=True), + ), + ] diff --git a/geonode/client/migrations/0002_auto_20180412_1039_squashed_0005_auto_20181015_1201.py b/geonode/client/migrations/0002_auto_20180412_1039_squashed_0005_auto_20181015_1201.py new file mode 100644 index 0000000..c842e75 --- /dev/null +++ b/geonode/client/migrations/0002_auto_20180412_1039_squashed_0005_auto_20181015_1201.py @@ -0,0 +1,45 @@ +# Generated by Django 1.11.20 on 2019-04-04 08:32 + + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('geonode_client', '0002_auto_20180412_1039'), ('geonode_client', '0003_geonodethemecustomization_jumbotron_welcome_hide'), ('geonode_client', '0004_auto_20180416_1319'), ('geonode_client', '0005_auto_20181015_1201')] + + dependencies = [ + ('geonode_client', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='geonodethemecustomization', + name='partners_title', + field=models.CharField(blank=True, default='Our Partners', max_length=100, null=True), + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_welcome_hide', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_site_description', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_welcome_content', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_welcome_title', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.DeleteModel( + name='GeoNodeThemeCustomization', + ), + ] diff --git a/geonode/client/migrations/0003_geonodethemecustomization_jumbotron_welcome_hide.py b/geonode/client/migrations/0003_geonodethemecustomization_jumbotron_welcome_hide.py new file mode 100644 index 0000000..3ad9548 --- /dev/null +++ b/geonode/client/migrations/0003_geonodethemecustomization_jumbotron_welcome_hide.py @@ -0,0 +1,19 @@ +# Generated by Django 1.11.11 on 2018-04-16 09:49 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geonode_client', '0002_auto_20180412_1039'), + ] + + operations = [ + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_welcome_hide', + field=models.BooleanField(default=False), + ), + ] diff --git a/geonode/client/migrations/0004_auto_20180416_1319.py b/geonode/client/migrations/0004_auto_20180416_1319.py new file mode 100644 index 0000000..30dd821 --- /dev/null +++ b/geonode/client/migrations/0004_auto_20180416_1319.py @@ -0,0 +1,29 @@ +# Generated by Django 1.11.11 on 2018-04-16 13:19 + + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geonode_client', '0003_geonodethemecustomization_jumbotron_welcome_hide'), + ] + + operations = [ + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_site_description', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_welcome_content', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='geonodethemecustomization', + name='jumbotron_welcome_title', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/geonode/client/migrations/0005_auto_20181015_1201.py b/geonode/client/migrations/0005_auto_20181015_1201.py new file mode 100644 index 0000000..54bccb2 --- /dev/null +++ b/geonode/client/migrations/0005_auto_20181015_1201.py @@ -0,0 +1,24 @@ +# Generated by Django 1.11.15 on 2018-10-15 00:01 + + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('geonode_client', '0004_auto_20180416_1319'), + ] + + operations = [ + migrations.RemoveField( + model_name='geonodethemecustomization', + name='partners', + ), + migrations.DeleteModel( + name='GeoNodeThemeCustomization', + ), + migrations.DeleteModel( + name='Partner', + ), + ] diff --git a/geonode/client/migrations/__init__.py b/geonode/client/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/client/models.py b/geonode/client/models.py new file mode 100644 index 0000000..68c7b7d --- /dev/null +++ b/geonode/client/models.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/client/static/crossdomain.xml b/geonode/client/static/crossdomain.xml new file mode 100644 index 0000000..981d5f6 --- /dev/null +++ b/geonode/client/static/crossdomain.xml @@ -0,0 +1,4 @@ + + + + diff --git a/geonode/client/static/yarn.lock b/geonode/client/static/yarn.lock new file mode 100644 index 0000000..f41eb7a --- /dev/null +++ b/geonode/client/static/yarn.lock @@ -0,0 +1,5200 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.13.tgz#dcfc826beef65e75c50e21d3837d7d95798dd658" + integrity sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g== + dependencies: + "@babel/highlight" "^7.12.13" + +"@babel/compat-data@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.12.13.tgz#27e19e0ed3726ccf54067ced4109501765e7e2e8" + integrity sha512-U/hshG5R+SIoW7HVWIdmy1cB7s3ki+r3FpyEZiCgpi4tFgPnX/vynY80ZGSASOIrUM6O7VxOgCZgdt7h97bUGg== + +"@babel/core@^7.8.6": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.13.tgz#b73a87a3a3e7d142a66248bf6ad88b9ceb093425" + integrity sha512-BQKE9kXkPlXHPeqissfxo0lySWJcYdEP0hdtJOH/iJfDdhOCcgtNCjftCJg3qqauB4h+lz2N6ixM++b9DN1Tcw== + dependencies: + "@babel/code-frame" "^7.12.13" + "@babel/generator" "^7.12.13" + "@babel/helper-module-transforms" "^7.12.13" + "@babel/helpers" "^7.12.13" + "@babel/parser" "^7.12.13" + "@babel/template" "^7.12.13" + "@babel/traverse" "^7.12.13" + "@babel/types" "^7.12.13" + convert-source-map "^1.7.0" + debug "^4.1.0" + gensync "^1.0.0-beta.1" + json5 "^2.1.2" + lodash "^4.17.19" + semver "^5.4.1" + source-map "^0.5.0" + +"@babel/generator@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.13.tgz#5f6ebe6c85db99886db2d7b044409196f872a503" + integrity sha512-9qQ8Fgo8HaSvHEt6A5+BATP7XktD/AdAnObUeTRz5/e2y3kbrxZgz32qUJJsdmwUvBJzF4AeV21nGTNwv05Mpw== + dependencies: + "@babel/types" "^7.12.13" + jsesc "^2.5.1" + source-map "^0.5.0" + +"@babel/helper-annotate-as-pure@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.13.tgz#0f58e86dfc4bb3b1fcd7db806570e177d439b6ab" + integrity sha512-7YXfX5wQ5aYM/BOlbSccHDbuXXFPxeoUmfWtz8le2yTkTZc+BxsiEnENFoi2SlmA8ewDkG2LgIMIVzzn2h8kfw== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-builder-binary-assignment-operator-visitor@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.12.13.tgz#6bc20361c88b0a74d05137a65cac8d3cbf6f61fc" + integrity sha512-CZOv9tGphhDRlVjVkAgm8Nhklm9RzSmWpX2my+t7Ua/KT616pEzXsQCjinzvkRvHWJ9itO4f296efroX23XCMA== + dependencies: + "@babel/helper-explode-assignable-expression" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/helper-compilation-targets@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.12.13.tgz#d689cdef88810aa74e15a7a94186f26a3d773c98" + integrity sha512-dXof20y/6wB5HnLOGyLh/gobsMvDNoekcC+8MCV2iaTd5JemhFkPD73QB+tK3iFC9P0xJC73B6MvKkyUfS9cCw== + dependencies: + "@babel/compat-data" "^7.12.13" + "@babel/helper-validator-option" "^7.12.11" + browserslist "^4.14.5" + semver "^5.5.0" + +"@babel/helper-create-class-features-plugin@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.12.13.tgz#0f1707c2eec1a4604f2a22a6fb209854ef2a399a" + integrity sha512-Vs/e9wv7rakKYeywsmEBSRC9KtmE7Px+YBlESekLeJOF0zbGUicGfXSNi3o+tfXSNS48U/7K9mIOOCR79Cl3+Q== + dependencies: + "@babel/helper-function-name" "^7.12.13" + "@babel/helper-member-expression-to-functions" "^7.12.13" + "@babel/helper-optimise-call-expression" "^7.12.13" + "@babel/helper-replace-supers" "^7.12.13" + "@babel/helper-split-export-declaration" "^7.12.13" + +"@babel/helper-create-regexp-features-plugin@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.13.tgz#0996d370a92896c612ae41a4215544bd152579c0" + integrity sha512-XC+kiA0J3at6E85dL5UnCYfVOcIZ834QcAY0TIpgUVnz0zDzg+0TtvZTnJ4g9L1dPRGe30Qi03XCIS4tYCLtqw== + dependencies: + "@babel/helper-annotate-as-pure" "^7.12.13" + regexpu-core "^4.7.1" + +"@babel/helper-explode-assignable-expression@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.12.13.tgz#0e46990da9e271502f77507efa4c9918d3d8634a" + integrity sha512-5loeRNvMo9mx1dA/d6yNi+YiKziJZFylZnCo1nmFF4qPU4yJ14abhWESuSMQSlQxWdxdOFzxXjk/PpfudTtYyw== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-function-name@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.12.13.tgz#93ad656db3c3c2232559fd7b2c3dbdcbe0eb377a" + integrity sha512-TZvmPn0UOqmvi5G4vvw0qZTpVptGkB1GL61R6lKvrSdIxGm5Pky7Q3fpKiIkQCAtRCBUwB0PaThlx9vebCDSwA== + dependencies: + "@babel/helper-get-function-arity" "^7.12.13" + "@babel/template" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/helper-get-function-arity@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.13.tgz#bc63451d403a3b3082b97e1d8b3fe5bd4091e583" + integrity sha512-DjEVzQNz5LICkzN0REdpD5prGoidvbdYk1BVgRUOINaWJP2t6avB27X1guXK1kXNrX0WMfsrm1A/ZBthYuIMQg== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-hoist-variables@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.12.13.tgz#13aba58b7480b502362316ea02f52cca0e9796cd" + integrity sha512-KSC5XSj5HreRhYQtZ3cnSnQwDzgnbdUDEFsxkN0m6Q3WrCRt72xrnZ8+h+pX7YxM7hr87zIO3a/v5p/H3TrnVw== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-member-expression-to-functions@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.13.tgz#c5715695b4f8bab32660dbdcdc2341dec7e3df40" + integrity sha512-B+7nN0gIL8FZ8SvMcF+EPyB21KnCcZHQZFczCxbiNGV/O0rsrSBlWGLzmtBJ3GMjSVMIm4lpFhR+VdVBuIsUcQ== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-module-imports@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.12.13.tgz#ec67e4404f41750463e455cc3203f6a32e93fcb0" + integrity sha512-NGmfvRp9Rqxy0uHSSVP+SRIW1q31a7Ji10cLBcqSDUngGentY4FRiHOFZFE1CLU5eiL0oE8reH7Tg1y99TDM/g== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-module-transforms@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.12.13.tgz#01afb052dcad2044289b7b20beb3fa8bd0265bea" + integrity sha512-acKF7EjqOR67ASIlDTupwkKM1eUisNAjaSduo5Cz+793ikfnpe7p4Q7B7EWU2PCoSTPWsQkR7hRUWEIZPiVLGA== + dependencies: + "@babel/helper-module-imports" "^7.12.13" + "@babel/helper-replace-supers" "^7.12.13" + "@babel/helper-simple-access" "^7.12.13" + "@babel/helper-split-export-declaration" "^7.12.13" + "@babel/helper-validator-identifier" "^7.12.11" + "@babel/template" "^7.12.13" + "@babel/traverse" "^7.12.13" + "@babel/types" "^7.12.13" + lodash "^4.17.19" + +"@babel/helper-optimise-call-expression@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.13.tgz#5c02d171b4c8615b1e7163f888c1c81c30a2aaea" + integrity sha512-BdWQhoVJkp6nVjB7nkFWcn43dkprYauqtk++Py2eaf/GRDFm5BxRqEIZCiHlZUGAVmtwKcsVL1dC68WmzeFmiA== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.12.13.tgz#174254d0f2424d8aefb4dd48057511247b0a9eeb" + integrity sha512-C+10MXCXJLiR6IeG9+Wiejt9jmtFpxUc3MQqCmPY8hfCjyUGl9kT+B2okzEZrtykiwrc4dbCPdDoz0A/HQbDaA== + +"@babel/helper-remap-async-to-generator@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.12.13.tgz#170365f4140e2d20e5c88f8ba23c24468c296878" + integrity sha512-Qa6PU9vNcj1NZacZZI1Mvwt+gXDH6CTfgAkSjeRMLE8HxtDK76+YDId6NQR+z7Rgd5arhD2cIbS74r0SxD6PDA== + dependencies: + "@babel/helper-annotate-as-pure" "^7.12.13" + "@babel/helper-wrap-function" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/helper-replace-supers@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.12.13.tgz#00ec4fb6862546bd3d0aff9aac56074277173121" + integrity sha512-pctAOIAMVStI2TMLhozPKbf5yTEXc0OJa0eENheb4w09SrgOWEs+P4nTOZYJQCqs8JlErGLDPDJTiGIp3ygbLg== + dependencies: + "@babel/helper-member-expression-to-functions" "^7.12.13" + "@babel/helper-optimise-call-expression" "^7.12.13" + "@babel/traverse" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/helper-simple-access@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.12.13.tgz#8478bcc5cacf6aa1672b251c1d2dde5ccd61a6c4" + integrity sha512-0ski5dyYIHEfwpWGx5GPWhH35j342JaflmCeQmsPWcrOQDtCN6C1zKAVRFVbK53lPW2c9TsuLLSUDf0tIGJ5hA== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-skip-transparent-expression-wrappers@^7.12.1": + version "7.12.1" + resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.12.1.tgz#462dc63a7e435ade8468385c63d2b84cce4b3cbf" + integrity sha512-Mf5AUuhG1/OCChOJ/HcADmvcHM42WJockombn8ATJG3OnyiSxBK/Mm5x78BQWvmtXZKHgbjdGL2kin/HOLlZGA== + dependencies: + "@babel/types" "^7.12.1" + +"@babel/helper-split-export-declaration@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.13.tgz#e9430be00baf3e88b0e13e6f9d4eaf2136372b05" + integrity sha512-tCJDltF83htUtXx5NLcaDqRmknv652ZWCHyoTETf1CXYJdPC7nohZohjUgieXhv0hTJdRf2FjDueFehdNucpzg== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-validator-identifier@^7.12.11": + version "7.12.11" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed" + integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== + +"@babel/helper-validator-option@^7.12.11": + version "7.12.11" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.12.11.tgz#d66cb8b7a3e7fe4c6962b32020a131ecf0847f4f" + integrity sha512-TBFCyj939mFSdeX7U7DDj32WtzYY7fDcalgq8v3fBZMNOJQNn7nOYzMaUCiPxPYfCup69mtIpqlKgMZLvQ8Xhw== + +"@babel/helper-wrap-function@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.12.13.tgz#e3ea8cb3ee0a16911f9c1b50d9e99fe8fe30f9ff" + integrity sha512-t0aZFEmBJ1LojdtJnhOaQEVejnzYhyjWHSsNSNo8vOYRbAJNh6r6GQF7pd36SqG7OKGbn+AewVQ/0IfYfIuGdw== + dependencies: + "@babel/helper-function-name" "^7.12.13" + "@babel/template" "^7.12.13" + "@babel/traverse" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/helpers@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.12.13.tgz#3c75e993632e4dadc0274eae219c73eb7645ba47" + integrity sha512-oohVzLRZ3GQEk4Cjhfs9YkJA4TdIDTObdBEZGrd6F/T0GPSnuV6l22eMcxlvcvzVIPH3VTtxbseudM1zIE+rPQ== + dependencies: + "@babel/template" "^7.12.13" + "@babel/traverse" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/highlight@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.12.13.tgz#8ab538393e00370b26271b01fa08f7f27f2e795c" + integrity sha512-kocDQvIbgMKlWxXe9fof3TQ+gkIPOUSEYhJjqUjvKMez3krV7vbzYCDq39Oj11UAVK7JqPVGQPlgE85dPNlQww== + dependencies: + "@babel/helper-validator-identifier" "^7.12.11" + chalk "^2.0.0" + js-tokens "^4.0.0" + +"@babel/parser@^7.12.13": + version "7.12.14" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.14.tgz#4adb7c5eef1d437ef965ad1569cd826db8c11dc9" + integrity sha512-xcfxDq3OrBnDsA/Z8eK5/2iPcLD8qbOaSSfOw4RA6jp4i7e6dEQ7+wTwxItEwzcXPQcsry5nZk96gmVPKletjQ== + +"@babel/plugin-proposal-async-generator-functions@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.13.tgz#d1c6d841802ffb88c64a2413e311f7345b9e66b5" + integrity sha512-1KH46Hx4WqP77f978+5Ye/VUbuwQld2hph70yaw2hXS2v7ER2f3nlpNMu909HO2rbvP0NKLlMVDPh9KXklVMhA== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-remap-async-to-generator" "^7.12.13" + "@babel/plugin-syntax-async-generators" "^7.8.0" + +"@babel/plugin-proposal-class-properties@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.12.13.tgz#3d2ce350367058033c93c098e348161d6dc0d8c8" + integrity sha512-8SCJ0Ddrpwv4T7Gwb33EmW1V9PY5lggTO+A8WjyIwxrSHDUyBw4MtF96ifn1n8H806YlxbVCoKXbbmzD6RD+cA== + dependencies: + "@babel/helper-create-class-features-plugin" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-proposal-dynamic-import@^7.12.1": + version "7.12.1" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.12.1.tgz#43eb5c2a3487ecd98c5c8ea8b5fdb69a2749b2dc" + integrity sha512-a4rhUSZFuq5W8/OO8H7BL5zspjnc1FLd9hlOxIK/f7qG4a0qsqk8uvF/ywgBA8/OmjsapjpvaEOYItfGG1qIvQ== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + "@babel/plugin-syntax-dynamic-import" "^7.8.0" + +"@babel/plugin-proposal-export-namespace-from@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.12.13.tgz#393be47a4acd03fa2af6e3cde9b06e33de1b446d" + integrity sha512-INAgtFo4OnLN3Y/j0VwAgw3HDXcDtX+C/erMvWzuV9v71r7urb6iyMXu7eM9IgLr1ElLlOkaHjJ0SbCmdOQ3Iw== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/plugin-syntax-export-namespace-from" "^7.8.3" + +"@babel/plugin-proposal-json-strings@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.12.13.tgz#ced7888a2db92a3d520a2e35eb421fdb7fcc9b5d" + integrity sha512-v9eEi4GiORDg8x+Dmi5r8ibOe0VXoKDeNPYcTTxdGN4eOWikrJfDJCJrr1l5gKGvsNyGJbrfMftC2dTL6oz7pg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/plugin-syntax-json-strings" "^7.8.0" + +"@babel/plugin-proposal-logical-assignment-operators@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.12.13.tgz#575b5d9a08d8299eeb4db6430da6e16e5cf14350" + integrity sha512-fqmiD3Lz7jVdK6kabeSr1PZlWSUVqSitmHEe3Z00dtGTKieWnX9beafvavc32kjORa5Bai4QNHgFDwWJP+WtSQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" + +"@babel/plugin-proposal-nullish-coalescing-operator@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.12.13.tgz#24867307285cee4e1031170efd8a7ac807deefde" + integrity sha512-Qoxpy+OxhDBI5kRqliJFAl4uWXk3Bn24WeFstPH0iLymFehSAUR8MHpqU7njyXv/qbo7oN6yTy5bfCmXdKpo1Q== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0" + +"@babel/plugin-proposal-numeric-separator@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.12.13.tgz#bd9da3188e787b5120b4f9d465a8261ce67ed1db" + integrity sha512-O1jFia9R8BUCl3ZGB7eitaAPu62TXJRHn7rh+ojNERCFyqRwJMTmhz+tJ+k0CwI6CLjX/ee4qW74FSqlq9I35w== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/plugin-syntax-numeric-separator" "^7.10.4" + +"@babel/plugin-proposal-object-rest-spread@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.13.tgz#f93f3116381ff94bc676fdcb29d71045cd1ec011" + integrity sha512-WvA1okB/0OS/N3Ldb3sziSrXg6sRphsBgqiccfcQq7woEn5wQLNX82Oc4PlaFcdwcWHuQXAtb8ftbS8Fbsg/sg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/plugin-syntax-object-rest-spread" "^7.8.0" + "@babel/plugin-transform-parameters" "^7.12.13" + +"@babel/plugin-proposal-optional-catch-binding@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.12.13.tgz#4640520afe57728af14b4d1574ba844f263bcae5" + integrity sha512-9+MIm6msl9sHWg58NvqpNpLtuFbmpFYk37x8kgnGzAHvX35E1FyAwSUt5hIkSoWJFSAH+iwU8bJ4fcD1zKXOzg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/plugin-syntax-optional-catch-binding" "^7.8.0" + +"@babel/plugin-proposal-optional-chaining@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.12.13.tgz#63a7d805bc8ce626f3234ee5421a2a7fb23f66d9" + integrity sha512-0ZwjGfTcnZqyV3y9DSD1Yk3ebp+sIUpT2YDqP8hovzaNZnQq2Kd7PEqa6iOIUDBXBt7Jl3P7YAcEIL5Pz8u09Q== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1" + "@babel/plugin-syntax-optional-chaining" "^7.8.0" + +"@babel/plugin-proposal-private-methods@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.12.13.tgz#ea78a12554d784ecf7fc55950b752d469d9c4a71" + integrity sha512-sV0V57uUwpauixvR7s2o75LmwJI6JECwm5oPUY5beZB1nBl2i37hc7CJGqB5G+58fur5Y6ugvl3LRONk5x34rg== + dependencies: + "@babel/helper-create-class-features-plugin" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-proposal-unicode-property-regex@^7.12.13", "@babel/plugin-proposal-unicode-property-regex@^7.4.4": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.12.13.tgz#bebde51339be829c17aaaaced18641deb62b39ba" + integrity sha512-XyJmZidNfofEkqFV5VC/bLabGmO5QzenPO/YOfGuEbgU+2sSwMmio3YLb4WtBgcmmdwZHyVyv8on77IUjQ5Gvg== + dependencies: + "@babel/helper-create-regexp-features-plugin" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-syntax-async-generators@^7.8.0": + version "7.8.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" + integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-class-properties@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz#b5c987274c4a3a82b89714796931a6b53544ae10" + integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-syntax-dynamic-import@^7.8.0": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz#62bf98b2da3cd21d626154fc96ee5b3cb68eacb3" + integrity sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-export-namespace-from@^7.8.3": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz#028964a9ba80dbc094c915c487ad7c4e7a66465a" + integrity sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q== + dependencies: + "@babel/helper-plugin-utils" "^7.8.3" + +"@babel/plugin-syntax-json-strings@^7.8.0": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz#01ca21b668cd8218c9e640cb6dd88c5412b2c96a" + integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-logical-assignment-operators@^7.10.4": + version "7.10.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz#ca91ef46303530448b906652bac2e9fe9941f699" + integrity sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.0": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz#167ed70368886081f74b5c36c65a88c03b66d1a9" + integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-numeric-separator@^7.10.4": + version "7.10.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz#b9b070b3e33570cd9fd07ba7fa91c0dd37b9af97" + integrity sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-object-rest-spread@^7.8.0": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871" + integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-optional-catch-binding@^7.8.0": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz#6111a265bcfb020eb9efd0fdfd7d26402b9ed6c1" + integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-optional-chaining@^7.8.0": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz#4f69c2ab95167e0180cd5336613f8c5788f7d48a" + integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-top-level-await@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.13.tgz#c5f0fa6e249f5b739727f923540cf7a806130178" + integrity sha512-A81F9pDwyS7yM//KwbCSDqy3Uj4NMIurtplxphWxoYtNPov7cJsDkAFNNyVlIZ3jwGycVsurZ+LtOA8gZ376iQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-arrow-functions@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.12.13.tgz#eda5670b282952100c229f8a3bd49e0f6a72e9fe" + integrity sha512-tBtuN6qtCTd+iHzVZVOMNp+L04iIJBpqkdY42tWbmjIT5wvR2kx7gxMBsyhQtFzHwBbyGi9h8J8r9HgnOpQHxg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-async-to-generator@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.12.13.tgz#fed8c69eebf187a535bfa4ee97a614009b24f7ae" + integrity sha512-psM9QHcHaDr+HZpRuJcE1PXESuGWSCcbiGFFhhwfzdbTxaGDVzuVtdNYliAwcRo3GFg0Bc8MmI+AvIGYIJG04A== + dependencies: + "@babel/helper-module-imports" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-remap-async-to-generator" "^7.12.13" + +"@babel/plugin-transform-block-scoped-functions@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.12.13.tgz#a9bf1836f2a39b4eb6cf09967739de29ea4bf4c4" + integrity sha512-zNyFqbc3kI/fVpqwfqkg6RvBgFpC4J18aKKMmv7KdQ/1GgREapSJAykLMVNwfRGO3BtHj3YQZl8kxCXPcVMVeg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-block-scoping@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.12.13.tgz#f36e55076d06f41dfd78557ea039c1b581642e61" + integrity sha512-Pxwe0iqWJX4fOOM2kEZeUuAxHMWb9nK+9oh5d11bsLoB0xMg+mkDpt0eYuDZB7ETrY9bbcVlKUGTOGWy7BHsMQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-classes@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.12.13.tgz#9728edc1838b5d62fc93ad830bd523b1fcb0e1f6" + integrity sha512-cqZlMlhCC1rVnxE5ZGMtIb896ijL90xppMiuWXcwcOAuFczynpd3KYemb91XFFPi3wJSe/OcrX9lXoowatkkxA== + dependencies: + "@babel/helper-annotate-as-pure" "^7.12.13" + "@babel/helper-function-name" "^7.12.13" + "@babel/helper-optimise-call-expression" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-replace-supers" "^7.12.13" + "@babel/helper-split-export-declaration" "^7.12.13" + globals "^11.1.0" + +"@babel/plugin-transform-computed-properties@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.12.13.tgz#6a210647a3d67f21f699cfd2a01333803b27339d" + integrity sha512-dDfuROUPGK1mTtLKyDPUavmj2b6kFu82SmgpztBFEO974KMjJT+Ytj3/oWsTUMBmgPcp9J5Pc1SlcAYRpJ2hRA== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-destructuring@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.12.13.tgz#fc56c5176940c5b41735c677124d1d20cecc9aeb" + integrity sha512-Dn83KykIFzjhA3FDPA1z4N+yfF3btDGhjnJwxIj0T43tP0flCujnU8fKgEkf0C1biIpSv9NZegPBQ1J6jYkwvQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-dotall-regex@^7.12.13", "@babel/plugin-transform-dotall-regex@^7.4.4": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.12.13.tgz#3f1601cc29905bfcb67f53910f197aeafebb25ad" + integrity sha512-foDrozE65ZFdUC2OfgeOCrEPTxdB3yjqxpXh8CH+ipd9CHd4s/iq81kcUpyH8ACGNEPdFqbtzfgzbT/ZGlbDeQ== + dependencies: + "@babel/helper-create-regexp-features-plugin" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-duplicate-keys@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.12.13.tgz#6f06b87a8b803fd928e54b81c258f0a0033904de" + integrity sha512-NfADJiiHdhLBW3pulJlJI2NB0t4cci4WTZ8FtdIuNc2+8pslXdPtRRAEWqUY+m9kNOk2eRYbTAOipAxlrOcwwQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-exponentiation-operator@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.12.13.tgz#4d52390b9a273e651e4aba6aee49ef40e80cd0a1" + integrity sha512-fbUelkM1apvqez/yYx1/oICVnGo2KM5s63mhGylrmXUxK/IAXSIf87QIxVfZldWf4QsOafY6vV3bX8aMHSvNrA== + dependencies: + "@babel/helper-builder-binary-assignment-operator-visitor" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-for-of@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.12.13.tgz#561ff6d74d9e1c8879cb12dbaf4a14cd29d15cf6" + integrity sha512-xCbdgSzXYmHGyVX3+BsQjcd4hv4vA/FDy7Kc8eOpzKmBBPEOTurt0w5fCRQaGl+GSBORKgJdstQ1rHl4jbNseQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-function-name@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.12.13.tgz#bb024452f9aaed861d374c8e7a24252ce3a50051" + integrity sha512-6K7gZycG0cmIwwF7uMK/ZqeCikCGVBdyP2J5SKNCXO5EOHcqi+z7Jwf8AmyDNcBgxET8DrEtCt/mPKPyAzXyqQ== + dependencies: + "@babel/helper-function-name" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-literals@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.12.13.tgz#2ca45bafe4a820197cf315794a4d26560fe4bdb9" + integrity sha512-FW+WPjSR7hiUxMcKqyNjP05tQ2kmBCdpEpZHY1ARm96tGQCCBvXKnpjILtDplUnJ/eHZ0lALLM+d2lMFSpYJrQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-member-expression-literals@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.12.13.tgz#5ffa66cd59b9e191314c9f1f803b938e8c081e40" + integrity sha512-kxLkOsg8yir4YeEPHLuO2tXP9R/gTjpuTOjshqSpELUN3ZAg2jfDnKUvzzJxObun38sw3wm4Uu69sX/zA7iRvg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-modules-amd@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.12.13.tgz#43db16249b274ee2e551e2422090aa1c47692d56" + integrity sha512-JHLOU0o81m5UqG0Ulz/fPC68/v+UTuGTWaZBUwpEk1fYQ1D9LfKV6MPn4ttJKqRo5Lm460fkzjLTL4EHvCprvA== + dependencies: + "@babel/helper-module-transforms" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + babel-plugin-dynamic-import-node "^2.3.3" + +"@babel/plugin-transform-modules-commonjs@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.12.13.tgz#5043b870a784a8421fa1fd9136a24f294da13e50" + integrity sha512-OGQoeVXVi1259HjuoDnsQMlMkT9UkZT9TpXAsqWplS/M0N1g3TJAn/ByOCeQu7mfjc5WpSsRU+jV1Hd89ts0kQ== + dependencies: + "@babel/helper-module-transforms" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-simple-access" "^7.12.13" + babel-plugin-dynamic-import-node "^2.3.3" + +"@babel/plugin-transform-modules-systemjs@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.12.13.tgz#351937f392c7f07493fc79b2118201d50404a3c5" + integrity sha512-aHfVjhZ8QekaNF/5aNdStCGzwTbU7SI5hUybBKlMzqIMC7w7Ho8hx5a4R/DkTHfRfLwHGGxSpFt9BfxKCoXKoA== + dependencies: + "@babel/helper-hoist-variables" "^7.12.13" + "@babel/helper-module-transforms" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-validator-identifier" "^7.12.11" + babel-plugin-dynamic-import-node "^2.3.3" + +"@babel/plugin-transform-modules-umd@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.12.13.tgz#26c66f161d3456674e344b4b1255de4d530cfb37" + integrity sha512-BgZndyABRML4z6ibpi7Z98m4EVLFI9tVsZDADC14AElFaNHHBcJIovflJ6wtCqFxwy2YJ1tJhGRsr0yLPKoN+w== + dependencies: + "@babel/helper-module-transforms" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-named-capturing-groups-regex@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.12.13.tgz#2213725a5f5bbbe364b50c3ba5998c9599c5c9d9" + integrity sha512-Xsm8P2hr5hAxyYblrfACXpQKdQbx4m2df9/ZZSQ8MAhsadw06+jW7s9zsSw6he+mJZXRlVMyEnVktJo4zjk1WA== + dependencies: + "@babel/helper-create-regexp-features-plugin" "^7.12.13" + +"@babel/plugin-transform-new-target@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.12.13.tgz#e22d8c3af24b150dd528cbd6e685e799bf1c351c" + integrity sha512-/KY2hbLxrG5GTQ9zzZSc3xWiOy379pIETEhbtzwZcw9rvuaVV4Fqy7BYGYOWZnaoXIQYbbJ0ziXLa/sKcGCYEQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-object-super@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.12.13.tgz#b4416a2d63b8f7be314f3d349bd55a9c1b5171f7" + integrity sha512-JzYIcj3XtYspZDV8j9ulnoMPZZnF/Cj0LUxPOjR89BdBVx+zYJI9MdMIlUZjbXDX+6YVeS6I3e8op+qQ3BYBoQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-replace-supers" "^7.12.13" + +"@babel/plugin-transform-parameters@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.12.13.tgz#461e76dfb63c2dfd327b8a008a9e802818ce9853" + integrity sha512-e7QqwZalNiBRHCpJg/P8s/VJeSRYgmtWySs1JwvfwPqhBbiWfOcHDKdeAi6oAyIimoKWBlwc8oTgbZHdhCoVZA== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-property-literals@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.12.13.tgz#4e6a9e37864d8f1b3bc0e2dce7bf8857db8b1a81" + integrity sha512-nqVigwVan+lR+g8Fj8Exl0UQX2kymtjcWfMOYM1vTYEKujeyv2SkMgazf2qNcK7l4SDiKyTA/nHCPqL4e2zo1A== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-regenerator@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.12.13.tgz#b628bcc9c85260ac1aeb05b45bde25210194a2f5" + integrity sha512-lxb2ZAvSLyJ2PEe47hoGWPmW22v7CtSl9jW8mingV4H2sEX/JOcrAj2nPuGWi56ERUm2bUpjKzONAuT6HCn2EA== + dependencies: + regenerator-transform "^0.14.2" + +"@babel/plugin-transform-reserved-words@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.12.13.tgz#7d9988d4f06e0fe697ea1d9803188aa18b472695" + integrity sha512-xhUPzDXxZN1QfiOy/I5tyye+TRz6lA7z6xaT4CLOjPRMVg1ldRf0LHw0TDBpYL4vG78556WuHdyO9oi5UmzZBg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-shorthand-properties@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.12.13.tgz#db755732b70c539d504c6390d9ce90fe64aff7ad" + integrity sha512-xpL49pqPnLtf0tVluuqvzWIgLEhuPpZzvs2yabUHSKRNlN7ScYU7aMlmavOeyXJZKgZKQRBlh8rHbKiJDraTSw== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-spread@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.12.13.tgz#ca0d5645abbd560719c354451b849f14df4a7949" + integrity sha512-dUCrqPIowjqk5pXsx1zPftSq4sT0aCeZVAxhdgs3AMgyaDmoUT0G+5h3Dzja27t76aUEIJWlFgPJqJ/d4dbTtg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1" + +"@babel/plugin-transform-sticky-regex@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.12.13.tgz#760ffd936face73f860ae646fb86ee82f3d06d1f" + integrity sha512-Jc3JSaaWT8+fr7GRvQP02fKDsYk4K/lYwWq38r/UGfaxo89ajud321NH28KRQ7xy1Ybc0VUE5Pz8psjNNDUglg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-template-literals@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.12.13.tgz#655037b07ebbddaf3b7752f55d15c2fd6f5aa865" + integrity sha512-arIKlWYUgmNsF28EyfmiQHJLJFlAJNYkuQO10jL46ggjBpeb2re1P9K9YGxNJB45BqTbaslVysXDYm/g3sN/Qg== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-typeof-symbol@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.12.13.tgz#785dd67a1f2ea579d9c2be722de8c84cb85f5a7f" + integrity sha512-eKv/LmUJpMnu4npgfvs3LiHhJua5fo/CysENxa45YCQXZwKnGCQKAg87bvoqSW1fFT+HA32l03Qxsm8ouTY3ZQ== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-unicode-escapes@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.12.13.tgz#840ced3b816d3b5127dd1d12dcedc5dead1a5e74" + integrity sha512-0bHEkdwJ/sN/ikBHfSmOXPypN/beiGqjo+o4/5K+vxEFNPRPdImhviPakMKG4x96l85emoa0Z6cDflsdBusZbw== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-transform-unicode-regex@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.12.13.tgz#b52521685804e155b1202e83fc188d34bb70f5ac" + integrity sha512-mDRzSNY7/zopwisPZ5kM9XKCfhchqIYwAKRERtEnhYscZB79VRekuRSoYbN0+KVe3y8+q1h6A4svXtP7N+UoCA== + dependencies: + "@babel/helper-create-regexp-features-plugin" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/preset-env@^7.8.6": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.12.13.tgz#3aa2d09cf7d255177538dff292ac9af29ad46525" + integrity sha512-JUVlizG8SoFTz4LmVUL8++aVwzwxcvey3N0j1tRbMAXVEy95uQ/cnEkmEKHN00Bwq4voAV3imQGnQvpkLAxsrw== + dependencies: + "@babel/compat-data" "^7.12.13" + "@babel/helper-compilation-targets" "^7.12.13" + "@babel/helper-module-imports" "^7.12.13" + "@babel/helper-plugin-utils" "^7.12.13" + "@babel/helper-validator-option" "^7.12.11" + "@babel/plugin-proposal-async-generator-functions" "^7.12.13" + "@babel/plugin-proposal-class-properties" "^7.12.13" + "@babel/plugin-proposal-dynamic-import" "^7.12.1" + "@babel/plugin-proposal-export-namespace-from" "^7.12.13" + "@babel/plugin-proposal-json-strings" "^7.12.13" + "@babel/plugin-proposal-logical-assignment-operators" "^7.12.13" + "@babel/plugin-proposal-nullish-coalescing-operator" "^7.12.13" + "@babel/plugin-proposal-numeric-separator" "^7.12.13" + "@babel/plugin-proposal-object-rest-spread" "^7.12.13" + "@babel/plugin-proposal-optional-catch-binding" "^7.12.13" + "@babel/plugin-proposal-optional-chaining" "^7.12.13" + "@babel/plugin-proposal-private-methods" "^7.12.13" + "@babel/plugin-proposal-unicode-property-regex" "^7.12.13" + "@babel/plugin-syntax-async-generators" "^7.8.0" + "@babel/plugin-syntax-class-properties" "^7.12.13" + "@babel/plugin-syntax-dynamic-import" "^7.8.0" + "@babel/plugin-syntax-export-namespace-from" "^7.8.3" + "@babel/plugin-syntax-json-strings" "^7.8.0" + "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" + "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0" + "@babel/plugin-syntax-numeric-separator" "^7.10.4" + "@babel/plugin-syntax-object-rest-spread" "^7.8.0" + "@babel/plugin-syntax-optional-catch-binding" "^7.8.0" + "@babel/plugin-syntax-optional-chaining" "^7.8.0" + "@babel/plugin-syntax-top-level-await" "^7.12.13" + "@babel/plugin-transform-arrow-functions" "^7.12.13" + "@babel/plugin-transform-async-to-generator" "^7.12.13" + "@babel/plugin-transform-block-scoped-functions" "^7.12.13" + "@babel/plugin-transform-block-scoping" "^7.12.13" + "@babel/plugin-transform-classes" "^7.12.13" + "@babel/plugin-transform-computed-properties" "^7.12.13" + "@babel/plugin-transform-destructuring" "^7.12.13" + "@babel/plugin-transform-dotall-regex" "^7.12.13" + "@babel/plugin-transform-duplicate-keys" "^7.12.13" + "@babel/plugin-transform-exponentiation-operator" "^7.12.13" + "@babel/plugin-transform-for-of" "^7.12.13" + "@babel/plugin-transform-function-name" "^7.12.13" + "@babel/plugin-transform-literals" "^7.12.13" + "@babel/plugin-transform-member-expression-literals" "^7.12.13" + "@babel/plugin-transform-modules-amd" "^7.12.13" + "@babel/plugin-transform-modules-commonjs" "^7.12.13" + "@babel/plugin-transform-modules-systemjs" "^7.12.13" + "@babel/plugin-transform-modules-umd" "^7.12.13" + "@babel/plugin-transform-named-capturing-groups-regex" "^7.12.13" + "@babel/plugin-transform-new-target" "^7.12.13" + "@babel/plugin-transform-object-super" "^7.12.13" + "@babel/plugin-transform-parameters" "^7.12.13" + "@babel/plugin-transform-property-literals" "^7.12.13" + "@babel/plugin-transform-regenerator" "^7.12.13" + "@babel/plugin-transform-reserved-words" "^7.12.13" + "@babel/plugin-transform-shorthand-properties" "^7.12.13" + "@babel/plugin-transform-spread" "^7.12.13" + "@babel/plugin-transform-sticky-regex" "^7.12.13" + "@babel/plugin-transform-template-literals" "^7.12.13" + "@babel/plugin-transform-typeof-symbol" "^7.12.13" + "@babel/plugin-transform-unicode-escapes" "^7.12.13" + "@babel/plugin-transform-unicode-regex" "^7.12.13" + "@babel/preset-modules" "^0.1.3" + "@babel/types" "^7.12.13" + core-js-compat "^3.8.0" + semver "^5.5.0" + +"@babel/preset-modules@^0.1.3": + version "0.1.4" + resolved "https://registry.yarnpkg.com/@babel/preset-modules/-/preset-modules-0.1.4.tgz#362f2b68c662842970fdb5e254ffc8fc1c2e415e" + integrity sha512-J36NhwnfdzpmH41M1DrnkkgAqhZaqr/NBdPfQ677mLzlaXo+oDiv1deyCDtgAhz8p328otdob0Du7+xgHGZbKg== + dependencies: + "@babel/helper-plugin-utils" "^7.0.0" + "@babel/plugin-proposal-unicode-property-regex" "^7.4.4" + "@babel/plugin-transform-dotall-regex" "^7.4.4" + "@babel/types" "^7.4.4" + esutils "^2.0.2" + +"@babel/runtime@^7.8.4": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.12.13.tgz#0a21452352b02542db0ffb928ac2d3ca7cb6d66d" + integrity sha512-8+3UMPBrjFa/6TtKi/7sehPKqfAm4g6K+YQjyyFOLUTxzOngcRZTlAVY8sc2CORJYqdHQY8gRPHmn+qo15rCBw== + dependencies: + regenerator-runtime "^0.13.4" + +"@babel/template@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.13.tgz#530265be8a2589dbb37523844c5bcb55947fb327" + integrity sha512-/7xxiGA57xMo/P2GVvdEumr8ONhFOhfgq2ihK3h1e6THqzTAkHbkXgB0xI9yeTfIUoH3+oAeHhqm/I43OTbbjA== + dependencies: + "@babel/code-frame" "^7.12.13" + "@babel/parser" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/traverse@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.13.tgz#689f0e4b4c08587ad26622832632735fb8c4e0c0" + integrity sha512-3Zb4w7eE/OslI0fTp8c7b286/cQps3+vdLW3UcwC8VSJC6GbKn55aeVVu2QJNuCDoeKyptLOFrPq8WqZZBodyA== + dependencies: + "@babel/code-frame" "^7.12.13" + "@babel/generator" "^7.12.13" + "@babel/helper-function-name" "^7.12.13" + "@babel/helper-split-export-declaration" "^7.12.13" + "@babel/parser" "^7.12.13" + "@babel/types" "^7.12.13" + debug "^4.1.0" + globals "^11.1.0" + lodash "^4.17.19" + +"@babel/types@^7.12.1", "@babel/types@^7.12.13", "@babel/types@^7.4.4": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.13.tgz#8be1aa8f2c876da11a9cf650c0ecf656913ad611" + integrity sha512-oKrdZTld2im1z8bDwTOQvUbxKwE+854zc16qWZQlcTqMN00pWxHQ4ZeOq0yDMnisOpRykH2/5Qqcrk/OlbAjiQ== + dependencies: + "@babel/helper-validator-identifier" "^7.12.11" + lodash "^4.17.19" + to-fast-properties "^2.0.0" + +"@mapbox/point-geometry@~0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz#8a83f9335c7860effa2eeeca254332aa0aeed8f2" + integrity sha1-ioP5M1x4YO/6Lu7KJUMyqgru2PI= + +"@mapbox/vector-tile@1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@mapbox/vector-tile/-/vector-tile-1.3.0.tgz#c495f972525befccefcd838f45ffa37ef3b70fe8" + integrity sha1-xJX5clJb78zvzYOPRf+jfvO3D+g= + dependencies: + "@mapbox/point-geometry" "~0.1.0" + +"@terraformer/arcgis@^2.0.7": + version "2.0.7" + resolved "https://registry.yarnpkg.com/@terraformer/arcgis/-/arcgis-2.0.7.tgz#617e6056a64b7328d4827702d64d0ccfc2e96d17" + integrity sha512-7jIQcnd8RnKsDt1IxmEjq9t7l5fnf0e5c+c1+1GjG4NR+TY0XkJcWtrBv0DWd65uAlDiQYMlEG8ls5dz1GQiuA== + dependencies: + "@terraformer/common" "^2.0.7" + +"@terraformer/common@^2.0.7": + version "2.0.7" + resolved "https://registry.yarnpkg.com/@terraformer/common/-/common-2.0.7.tgz#bec626f331b9f0cc3aefb16f9f88e2d3684107ff" + integrity sha512-8bl+/JT0Rw6FYe2H3FfJS8uQwgzGl+UHs+8JX0TQLHgA4sMDEwObbMwo0iP3FVONwPXrPHEpC5YH7Grve0cl9A== + +"@turf/area@^5.1.5": + version "5.1.5" + resolved "https://registry.yarnpkg.com/@turf/area/-/area-5.1.5.tgz#efd899bfd260cdbd1541b2a3c155f8a5d2eefa1d" + integrity sha1-79iZv9Jgzb0VQbKjwVX4pdLu+h0= + dependencies: + "@turf/helpers" "^5.1.5" + "@turf/meta" "^5.1.5" + +"@turf/distance@^5.1.5": + version "5.1.5" + resolved "https://registry.yarnpkg.com/@turf/distance/-/distance-5.1.5.tgz#39cf18204bbf87587d707e609a60118909156409" + integrity sha1-Oc8YIEu/h1h9cH5gmmARiQkVZAk= + dependencies: + "@turf/helpers" "^5.1.5" + "@turf/invariant" "^5.1.5" + +"@turf/helpers@^5.1.5": + version "5.1.5" + resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-5.1.5.tgz#153405227ab933d004a5bb9641a9ed999fcbe0cf" + integrity sha1-FTQFInq5M9AEpbuWQantmZ/L4M8= + +"@turf/invariant@^5.1.5": + version "5.2.0" + resolved "https://registry.yarnpkg.com/@turf/invariant/-/invariant-5.2.0.tgz#f0150ff7290b38577b73d088b7932c1ee0aa90a7" + integrity sha1-8BUP9ykLOFd7c9CIt5MsHuCqkKc= + dependencies: + "@turf/helpers" "^5.1.5" + +"@turf/length@^5.1.5": + version "5.1.5" + resolved "https://registry.yarnpkg.com/@turf/length/-/length-5.1.5.tgz#f3a5f864c2b996a8bb471794535a1faf12eebefb" + integrity sha1-86X4ZMK5lqi7RxeUU1ofrxLuvvs= + dependencies: + "@turf/distance" "^5.1.5" + "@turf/helpers" "^5.1.5" + "@turf/meta" "^5.1.5" + +"@turf/meta@^5.1.5": + version "5.2.0" + resolved "https://registry.yarnpkg.com/@turf/meta/-/meta-5.2.0.tgz#3b1ad485ee0c3b0b1775132a32c384d53e4ba53d" + integrity sha1-OxrUhe4MOwsXdRMqMsOE1T5LpT0= + dependencies: + "@turf/helpers" "^5.1.5" + +"@types/minimatch@^3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" + integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== + +"@yaireo/tagify@^3.6.2": + version "3.22.1" + resolved "https://registry.yarnpkg.com/@yaireo/tagify/-/tagify-3.22.1.tgz#cbb3ed617b6af7b887ac29f14762ab14ccf7ca4a" + integrity sha512-pwokZQyW0qzyC85QYUy9rD4nQyEK8CsEHrCKI73F9kp5yL9g5nQspnKBgJJEyI/AFHyAes4hVAkzrpeuJ0RHrg== + +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== + +accepts@~1.3.4: + version "1.3.7" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" + integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== + dependencies: + mime-types "~2.1.24" + negotiator "0.6.2" + +acorn-node@^1.6.1: + version "1.8.2" + resolved "https://registry.yarnpkg.com/acorn-node/-/acorn-node-1.8.2.tgz#114c95d64539e53dede23de8b9d96df7c7ae2af8" + integrity sha512-8mt+fslDufLYntIoPAaIMUe/lrbrehIiwmR3t2k9LljIzoigEPF27eLk2hy8zSGzmR/ogr7zbRKINMo1u0yh5A== + dependencies: + acorn "^7.0.0" + acorn-walk "^7.0.0" + xtend "^4.0.2" + +acorn-walk@^7.0.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" + integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== + +acorn@4.x: + version "4.0.13" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-4.0.13.tgz#105495ae5361d697bd195c825192e1ad7f253787" + integrity sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c= + +acorn@5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.0.3.tgz#c460df08491463f028ccb82eab3730bf01087b3d" + integrity sha1-xGDfCEkUY/AozLguqzcwvwEIez0= + +acorn@^5.2.1: + version "5.7.4" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.4.tgz#3e8d8a9947d0599a1796d10225d7432f4a4acf5e" + integrity sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg== + +acorn@^7.0.0: + version "7.4.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" + integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== + +after@0.8.2: + version "0.8.2" + resolved "https://registry.yarnpkg.com/after/-/after-0.8.2.tgz#fedb394f9f0e02aa9768e702bda23b505fae7e1f" + integrity sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8= + +ajv@^4.9.1: + version "4.11.8" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" + integrity sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY= + dependencies: + co "^4.6.0" + json-stable-stringify "^1.0.1" + +align-text@^0.1.1, align-text@^0.1.3: + version "0.1.4" + resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" + integrity sha1-DNkKVhCT810KmSVsIrcGlDP60Rc= + dependencies: + kind-of "^3.0.2" + longest "^1.0.1" + repeat-string "^1.5.2" + +amdefine@>=0.0.4: + version "1.0.1" + resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5" + integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU= + +angular-cookies@1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/angular-cookies/-/angular-cookies-1.4.0.tgz#c520d6c9cc46d7c76f68c3b6b8aea5d561e90937" + integrity sha1-xSDWycxG18dvaMO2uK6l1WHpCTc= + +angular-leaflet-directive@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/angular-leaflet-directive/-/angular-leaflet-directive-0.10.0.tgz#ec2059f4c1017d5be720ba9a9a1e05801ea017a1" + integrity sha1-7CBZ9MEBfVvnILqamh4FgB6gF6E= + dependencies: + angular "1.x" + leaflet "0.7.x" + +angular-openlayers-directive@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/angular-openlayers-directive/-/angular-openlayers-directive-2.1.0.tgz#d272dd4ee5036f8d18dc54aa326968d10547a87f" + integrity sha1-0nLdTuUDb40Y3FSqMmlo0QVHqH8= + dependencies: + angular "~1.4.8" + angular-sanitize "~1.4.8" + openlayers "4.3.4" + +angular-sanitize@^1.7.9: + version "1.8.2" + resolved "https://registry.yarnpkg.com/angular-sanitize/-/angular-sanitize-1.8.2.tgz#ae78040f00c5e2ce1c63780bcc47fa14a1698296" + integrity sha512-OB6Goa+QN3byf5asQ7XRl7DKZejm/F/ZOqa9z1skqYVOWA2hoBxoCmt9E7+i7T/TbxZP5zYzKxNZVVJNu860Hg== + +angular-sanitize@~1.4.8: + version "1.4.14" + resolved "https://registry.yarnpkg.com/angular-sanitize/-/angular-sanitize-1.4.14.tgz#778c4e24941fa982d86cf7ebac3e613fa5d414fc" + integrity sha1-d4xOJJQfqYLYbPfrrD5hP6XUFPw= + +angular@1.x, angular@^1.8.2: + version "1.8.2" + resolved "https://registry.yarnpkg.com/angular/-/angular-1.8.2.tgz#5983bbb5a9fa63e213cb7749199e0d352de3a2f1" + integrity sha512-IauMOej2xEe7/7Ennahkbb5qd/HFADiNuLSESz9Q27inmi32zB0lnAsFeLEWcox3Gd1F6YhNd1CP7/9IukJ0Gw== + +angular@~1.4.8: + version "1.4.14" + resolved "https://registry.yarnpkg.com/angular/-/angular-1.4.14.tgz#9d0feaf60ce6e52ce50f49ee328656d1e1875c37" + integrity sha1-nQ/q9gzm5SzlD0nuMoZW0eGHXDc= + +ansi-regex@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" + integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= + +ansi-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" + integrity sha1-7QMXwyIGT3lGbAKWa922Bas32Zg= + +ansi-styles@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" + integrity sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4= + +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +ansi-styles@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-1.0.0.tgz#cb102df1c56f5123eab8b67cd7b98027a0279178" + integrity sha1-yxAt8cVvUSPquLZ817mAJ6AnkXg= + +aproba@^1.0.3: + version "1.2.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" + integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== + +are-we-there-yet@~1.1.2: + version "1.1.5" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" + integrity sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w== + dependencies: + delegates "^1.0.0" + readable-stream "^2.0.6" + +argparse@^1.0.7: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + +arr-diff@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520" + integrity sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA= + +arr-flatten@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/arr-flatten/-/arr-flatten-1.1.0.tgz#36048bbff4e7b47e136644316c99669ea5ae91f1" + integrity sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg== + +arr-union@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4" + integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ= + +array-differ@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b" + integrity sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg== + +array-each@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/array-each/-/array-each-1.0.1.tgz#a794af0c05ab1752846ee753a1f211a05ba0c44f" + integrity sha1-p5SvDAWrF1KEbudTofIRoFugxE8= + +array-slice@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/array-slice/-/array-slice-1.1.0.tgz#e368ea15f89bc7069f7ffb89aec3a6c7d4ac22d4" + integrity sha512-B1qMD3RBP7O8o0H2KbrXDyB0IccejMF15+87Lvlor12ONPRHP6gTjXMNkt/d3ZuOGbAe66hFmaCfECI24Ufp6w== + +array-union@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39" + integrity sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk= + dependencies: + array-uniq "^1.0.1" + +array-union@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" + integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== + +array-uniq@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6" + integrity sha1-r2rId6Jcx/dOBYiUdThY39sk/bY= + +array-unique@^0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" + integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= + +arraybuffer.slice@~0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675" + integrity sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog== + +arrify@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + +asn1@~0.2.3: + version "0.2.4" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" + integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= + +assert-plus@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234" + integrity sha1-104bh+ev/A24qttwIfP+SBAasjQ= + +assign-symbols@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" + integrity sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c= + +async-limiter@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" + integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== + +async@2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/async/-/async-2.5.0.tgz#843190fd6b7357a0b9e1c956edddd5ec8462b54d" + integrity sha512-e+lJAJeNWuPCNyxZKOBdaJGyLGHugXVQtrAwtuAe2vhxTYxFTKE73p8JuTmdH0qdQZtDvI4dhJwjZc5zsfIsYw== + dependencies: + lodash "^4.14.0" + +async@^1.4.0, async@~1.5.2: + version "1.5.2" + resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" + integrity sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo= + +async@^2.0.0, async@^2.6.0, async@^2.6.1: + version "2.6.3" + resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" + integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg== + dependencies: + lodash "^4.17.14" + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + +atob@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" + integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== + +aws-sign2@~0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" + integrity sha1-FDQt0428yU0OW4fXY81jYSwOeU8= + +aws4@^1.2.1: + version "1.11.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== + +babel-plugin-dynamic-import-node@^2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz#84fda19c976ec5c6defef57f9427b3def66e17a3" + integrity sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ== + dependencies: + object.assign "^4.1.0" + +babylon@~7.0.0-beta.19: + version "7.0.0-beta.47" + resolved "https://registry.yarnpkg.com/babylon/-/babylon-7.0.0-beta.47.tgz#6d1fa44f0abec41ab7c780481e62fd9aafbdea80" + integrity sha512-+rq2cr4GDhtToEzKFD6KZZMDBXhjFAr9JjPw9pAppZACeEWqNM294j+NdBzkSHYXwzzBmVjZ3nEVJlOhbR2gOQ== + +backo2@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/backo2/-/backo2-1.0.2.tgz#31ab1ac8b129363463e35b3ebb69f4dfcfba7947" + integrity sha1-MasayLEpNjRj41s+u2n038+6eUc= + +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + +base64-arraybuffer@0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz#73926771923b5a19747ad666aa5cd4bf9c6e9ce8" + integrity sha1-c5JncZI7Whl0etZmqlzUv5xunOg= + +base64id@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/base64id/-/base64id-1.0.0.tgz#47688cb99bb6804f0e06d3e763b1c32e57d8e6b6" + integrity sha1-R2iMuZu2gE8OBtPnY7HDLlfY5rY= + +base@^0.11.1: + version "0.11.2" + resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" + integrity sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg== + dependencies: + cache-base "^1.0.1" + class-utils "^0.3.5" + component-emitter "^1.2.1" + define-property "^1.0.0" + isobject "^3.0.1" + mixin-deep "^1.2.0" + pascalcase "^0.1.1" + +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= + dependencies: + tweetnacl "^0.14.3" + +better-assert@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/better-assert/-/better-assert-1.0.2.tgz#40866b9e1b9e0b55b481894311e68faffaebc522" + integrity sha1-QIZrnhueC1W0gYlDEeaPr/rrxSI= + dependencies: + callsite "1.0.0" + +binary@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79" + integrity sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk= + dependencies: + buffers "~0.1.1" + chainsaw "~0.1.0" + +blob@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/blob/-/blob-0.0.5.tgz#d680eeef25f8cd91ad533f5b01eed48e64caf683" + integrity sha512-gaqbzQPqOoamawKg0LGVd7SzLgXS+JH61oWprSLH+P+abTczqJbhTR8CmJ2u9/bUYNmHTGJx/UEmn6doAvvuig== + +bluebird@~3.5.0: + version "3.5.5" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.5.tgz#a8d0afd73251effbbd5fe384a77d73003c17a71f" + integrity sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w== + +body@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/body/-/body-5.1.0.tgz#e4ba0ce410a46936323367609ecb4e6553125069" + integrity sha1-5LoM5BCkaTYyM2dgnstOZVMSUGk= + dependencies: + continuable-cache "^0.3.1" + error "^7.0.0" + raw-body "~1.1.0" + safe-json-parse "~1.0.1" + +boom@2.x.x: + version "2.10.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f" + integrity sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8= + dependencies: + hoek "2.x.x" + +bootstrap-multiselect@^0.9.13-1: + version "0.9.15" + resolved "https://registry.yarnpkg.com/bootstrap-multiselect/-/bootstrap-multiselect-0.9.15.tgz#671db3bbe8d65d4caadc77ad12e10d65bf9eff71" + integrity sha512-UwF32a0QR82xkEEGpuNrn57Bu0b/7DfCuoiOaziSHfQKFj5arR6c7+MYLs5RiIf3zl4XZ+YnY7ZBi6/EN3vEZA== + dependencies: + jquery "~2.1.3" + +bootstrap-select@^1.13.10: + version "1.13.18" + resolved "https://registry.yarnpkg.com/bootstrap-select/-/bootstrap-select-1.13.18.tgz#4557119d58dc1159189977161c803962220e4dda" + integrity sha512-V1IzK4rxBq5FrJtkzSH6RmFLFBsjx50byFbfAf8jYyXROWs7ZpprGjdHeoyq2HSsHyjJhMMwjsQhRoYAfxCGow== + +bootstrap-table@^1.14.2: + version "1.18.2" + resolved "https://registry.yarnpkg.com/bootstrap-table/-/bootstrap-table-1.18.2.tgz#5b65fabe9d1462db193e9c844965da00aef5806c" + integrity sha512-BShrYY9bcadwxikP5Sd/+tZlbLcYqOBjYm5bdJLu3lRTgXEQ1p937ZNlcCMhIrRhvelUKlSl7EFORnEkSHR7gA== + +bootstrap-toggle@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/bootstrap-toggle/-/bootstrap-toggle-2.2.2.tgz#2b88534fc1b998674f877f98ba0d8b5b743e96fe" + integrity sha1-K4hTT8G5mGdPh3+Yug2LW3Q+lv4= + +bootstrap-tokenfield@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/bootstrap-tokenfield/-/bootstrap-tokenfield-0.12.0.tgz#5d763c3d196de71078ac4d49dc2ea169cb870cdf" + integrity sha1-XXY8PRlt5xB4rE1J3C6hacuHDN8= + dependencies: + jquery "^2.1.0" + +bootstrap-typeahead@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/bootstrap-typeahead/-/bootstrap-typeahead-2.3.2.tgz#462c3be72438c13d8ab6703ace8044cf37401236" + integrity sha1-Riw75yQ4wT2KtnA6zoBEzzdAEjY= + +bootstrap@3.4.1, bootstrap@^3.3, bootstrap@^3.4.1: + version "3.4.1" + resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.4.1.tgz#c3a347d419e289ad11f4033e3c4132b87c081d72" + integrity sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA== + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +braces@^2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729" + integrity sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w== + dependencies: + arr-flatten "^1.1.0" + array-unique "^0.3.2" + extend-shallow "^2.0.1" + fill-range "^4.0.0" + isobject "^3.0.1" + repeat-element "^1.1.2" + snapdragon "^0.8.1" + snapdragon-node "^2.0.1" + split-string "^3.0.2" + to-regex "^3.0.1" + +browser-split@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/browser-split/-/browser-split-0.0.0.tgz#41419caef769755929dd518967d3eec0a6262771" + integrity sha1-QUGcrvdpdVkp3VGJZ9PuwKYmJ3E= + +browserslist@^4.14.5, browserslist@^4.16.1: + version "4.16.3" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.3.tgz#340aa46940d7db878748567c5dea24a48ddf3717" + integrity sha512-vIyhWmIkULaq04Gt93txdh+j02yX/JzlyhLYbV3YQCn/zvES3JnY7TifHHvvr1w5hTDluNKMkV05cs4vy8Q7sw== + dependencies: + caniuse-lite "^1.0.30001181" + colorette "^1.2.1" + electron-to-chromium "^1.3.649" + escalade "^3.1.1" + node-releases "^1.1.70" + +buffers@~0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb" + integrity sha1-skV5w77U1tOWru5tmorn9Ugqt7s= + +builtin-modules@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-2.0.0.tgz#60b7ef5ae6546bd7deefa74b08b62a43a232648e" + integrity sha512-3U5kUA5VPsRUA3nofm/BXX7GVHKfxz0hOBAPxXrIvHzlDRkQVqEn6yi8QJegxl4LzOHLdvb7XF5dVawa/VVYBg== + +bytes@1: + version "1.0.0" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-1.0.0.tgz#3569ede8ba34315fab99c3e92cb04c7220de1fa8" + integrity sha1-NWnt6Lo0MV+rmcPpLLBMciDeH6g= + +cache-base@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2" + integrity sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ== + dependencies: + collection-visit "^1.0.0" + component-emitter "^1.2.1" + get-value "^2.0.6" + has-value "^1.0.0" + isobject "^3.0.1" + set-value "^2.0.0" + to-object-path "^0.3.0" + union-value "^1.0.0" + unset-value "^1.0.0" + +call-bind@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + +callsite@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/callsite/-/callsite-1.0.0.tgz#280398e5d664bd74038b6f0905153e6e8af1bc20" + integrity sha1-KAOY5dZkvXQDi28JBRU+borxvCA= + +camelcase@^1.0.2: + version "1.2.1" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-1.2.1.tgz#9bb5304d2e0b56698b2c758b08a3eaa9daa58a39" + integrity sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk= + +caniuse-lite@^1.0.30001181: + version "1.0.30001183" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001183.tgz#7a57ba9d6584119bb5f2bc76d3cc47ba9356b3e2" + integrity sha512-7JkwTEE1hlRKETbCFd8HDZeLiQIUcl8rC6JgNjvHCNaxOeNmQ9V4LvQXRUsKIV2CC73qKxljwVhToaA3kLRqTw== + +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= + +catharsis@~0.8.9: + version "0.8.11" + resolved "https://registry.yarnpkg.com/catharsis/-/catharsis-0.8.11.tgz#d0eb3d2b82b7da7a3ce2efb1a7b00becc6643468" + integrity sha512-a+xUyMV7hD1BrDQA/3iPV7oc+6W26BgVJO05PGEoatMyIuPScQKsde6i3YorWX1qs+AZjnJ18NqdKoCtKiNh1g== + dependencies: + lodash "^4.17.14" + +center-align@^0.1.1: + version "0.1.3" + resolved "https://registry.yarnpkg.com/center-align/-/center-align-0.1.3.tgz#aa0d32629b6ee972200411cbd4461c907bc2b7ad" + integrity sha1-qg0yYptu6XIgBBHL1EYckHvCt60= + dependencies: + align-text "^0.1.3" + lazy-cache "^1.0.3" + +chainsaw@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98" + integrity sha1-XqtQsor+WAdNDVgpE4iCi15fvJg= + dependencies: + traverse ">=0.3.0 <0.4" + +chalk@^1.0.0, chalk@^1.1.1: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" + integrity sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg= + dependencies: + ansi-styles "^2.2.1" + escape-string-regexp "^1.0.2" + has-ansi "^2.0.0" + strip-ansi "^3.0.0" + supports-color "^2.0.0" + +chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +chalk@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-0.4.0.tgz#5199a3ddcd0c1efe23bc08c1b027b06176e0c64f" + integrity sha1-UZmj3c0MHv4jvAjBsCewYXbgxk8= + dependencies: + ansi-styles "~1.0.0" + has-color "~0.1.0" + strip-ansi "~0.1.0" + +chalk@~4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" + integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +class-list@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/class-list/-/class-list-0.1.1.tgz#9b9745192c4179b5da0a0d7633658e3c70d796cb" + integrity sha1-m5dFGSxBebXaCg12M2WOPHDXlss= + dependencies: + indexof "0.0.1" + +class-utils@^0.3.5: + version "0.3.6" + resolved "https://registry.yarnpkg.com/class-utils/-/class-utils-0.3.6.tgz#f93369ae8b9a7ce02fd41faad0ca83033190c463" + integrity sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg== + dependencies: + arr-union "^3.1.0" + define-property "^0.2.5" + isobject "^3.0.0" + static-extend "^0.1.1" + +clean-css@~4.2.1: + version "4.2.3" + resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-4.2.3.tgz#507b5de7d97b48ee53d84adb0160ff6216380f78" + integrity sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA== + dependencies: + source-map "~0.6.0" + +cli@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cli/-/cli-1.0.1.tgz#22817534f24bfa4950c34d532d48ecbc621b8c14" + integrity sha1-IoF1NPJL+klQw01TLUjsvGIbjBQ= + dependencies: + exit "0.1.2" + glob "^7.1.1" + +clipboard@^2.0.4: + version "2.0.6" + resolved "https://registry.yarnpkg.com/clipboard/-/clipboard-2.0.6.tgz#52921296eec0fdf77ead1749421b21c968647376" + integrity sha512-g5zbiixBRk/wyKakSwCKd7vQXDjFnAMGHoEyBogG/bw9kTD9GvdAvaoRR1ALcEzt3pVKxZR0pViekPMIS0QyGg== + dependencies: + good-listener "^1.2.2" + select "^1.1.2" + tiny-emitter "^2.0.0" + +cliui@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-2.1.0.tgz#4b475760ff80264c762c3a1719032e91c7fea0d1" + integrity sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE= + dependencies: + center-align "^0.1.1" + right-align "^0.1.1" + wordwrap "0.0.2" + +closure-util@1.22.0: + version "1.22.0" + resolved "https://registry.yarnpkg.com/closure-util/-/closure-util-1.22.0.tgz#d2743107969a520be3c42523d5da380ab63ce2be" + integrity sha512-DX5IhSawO9838/fKNaH3Cx8jeeJGWORodHhTFkxCOVsr71OtXCfo/rtXsV24eq3iq31jdmgUQvlNIXEMHTlkWg== + dependencies: + acorn "5.0.3" + async "2.5.0" + fs-extra "3.0.1" + gaze "1.1.2" + get-down "1.2.0" + glob "7.1.2" + graceful-fs "4.1.11" + handlebars "4.0.10" + lodash "4.17.4" + minimatch "3.0.4" + mkdirp "0.5.1" + nomnom "1.8.1" + npmlog "4.1.2" + rimraf "2.6.1" + send "0.15.3" + socket.io "2.0.1" + temp "0.8.3" + +co@^4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" + integrity sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ= + +code-point-at@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" + integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= + +collection-visit@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" + integrity sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA= + dependencies: + map-visit "^1.0.0" + object-visit "^1.0.0" + +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +colorette@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" + integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== + +colors@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63" + integrity sha1-FopHAXVran9RoSzgyXv6KMCE7WM= + +combined-stream@^1.0.5, combined-stream@~1.0.5: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + +component-bind@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/component-bind/-/component-bind-1.0.0.tgz#00c608ab7dcd93897c0009651b1d3a8e1e73bbd1" + integrity sha1-AMYIq33Nk4l8AAllGx06jh5zu9E= + +component-emitter@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" + integrity sha1-E3kY1teCg/ffemt8WmPhQOaUJeY= + +component-emitter@^1.2.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" + integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== + +component-inherit@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/component-inherit/-/component-inherit-0.0.3.tgz#645fc4adf58b72b649d5cae65135619db26ff143" + integrity sha1-ZF/ErfWLcrZJ1crmUTVhnbJv8UM= + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + +console-browserify@1.1.x: + version "1.1.0" + resolved "https://registry.yarnpkg.com/console-browserify/-/console-browserify-1.1.0.tgz#f0241c45730a9fc6323b206dbf38edc741d0bb10" + integrity sha1-8CQcRXMKn8YyOyBtvzjtx0HQuxA= + dependencies: + date-now "^0.1.4" + +console-control-strings@^1.0.0, console-control-strings@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" + integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= + +continuable-cache@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/continuable-cache/-/continuable-cache-0.3.1.tgz#bd727a7faed77e71ff3985ac93351a912733ad0f" + integrity sha1-vXJ6f67XfnH/OYWskzUakSczrQ8= + +convert-source-map@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.7.0.tgz#17a2cb882d7f77d3490585e2ce6c524424a3a442" + integrity sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA== + dependencies: + safe-buffer "~5.1.1" + +cookie@0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.3.1.tgz#e7e0a1f9ef43b4c8ba925c5c5a96e806d16873bb" + integrity sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s= + +copy-anything@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/copy-anything/-/copy-anything-2.0.1.tgz#2afbce6da684bdfcbec93752fa762819cb480d9a" + integrity sha512-lA57e7viQHOdPQcrytv5jFeudZZOXuyk47lZym279FiDQ8jeZomXiGuVf6ffMKkJ+3TIai3J1J3yi6M+/4U35g== + dependencies: + is-what "^3.7.1" + +copy-descriptor@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" + integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= + +core-js-compat@^3.8.0: + version "3.8.3" + resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.8.3.tgz#9123fb6b9cad30f0651332dc77deba48ef9b0b3f" + integrity sha512-1sCb0wBXnBIL16pfFG1Gkvei6UzvKyTNYpiC41yrdjEv0UoJoq9E/abTMzyYJ6JpTkAj15dLjbqifIzEBDVvog== + dependencies: + browserslist "^4.16.1" + semver "7.0.0" + +core-util-is@1.0.2, core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +cropperjs@^1.5.6: + version "1.5.9" + resolved "https://registry.yarnpkg.com/cropperjs/-/cropperjs-1.5.9.tgz#4ff9d31e02ad04d2fc5df0044207c2ad53d99da8" + integrity sha512-aPWlg43sLIcYN4GBXIdyvM09wNPgn1ug+vNVwV8jlb3dbgEX/B34Iw6hrjGSajkUDQBmaCi6uPOevFb7N0yUsw== + +cryptiles@2.x.x: + version "2.0.5" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8" + integrity sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g= + dependencies: + boom "2.x.x" + +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= + dependencies: + assert-plus "^1.0.0" + +datatables@^1.10.18: + version "1.10.18" + resolved "https://registry.yarnpkg.com/datatables/-/datatables-1.10.18.tgz#fee16e82aa70b17c5faf1a6954ac68f404f33a70" + integrity sha512-ntatMgS9NN6UMpwbmO+QkYJuKlVeMA2Mi0Gu/QxyIh+dW7ZjLSDhPT2tWlzjpIWEkDYgieDzS9Nu7bdQCW0sbQ== + dependencies: + jquery ">=1.7" + +date-now@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" + integrity sha1-6vQ5/U1ISK105cx9vvIAZyueNFs= + +dateformat@~3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" + integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q== + +debug@2.6.4: + version "2.6.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.4.tgz#7586a9b3c39741c0282ae33445c4e8ac74734fe0" + integrity sha1-dYaps8OXQcAoKuM0RcTorHRzT+A= + dependencies: + ms "0.7.3" + +debug@2.6.7: + version "2.6.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.7.tgz#92bad1f6d05bbb6bba22cca88bcd0ec894c2861e" + integrity sha1-krrR9tBbu2u6Isyoi80OyJTChh4= + dependencies: + ms "2.0.0" + +debug@^2.2.0, debug@^2.3.3, debug@~2.6.6: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +debug@^3.1.0: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + +debug@^4.1.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" + integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + dependencies: + ms "2.1.2" + +debug@~3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== + dependencies: + ms "2.0.0" + +decamelize@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" + integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= + +decode-uri-component@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" + integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= + +decompress-zip@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/decompress-zip/-/decompress-zip-0.3.0.tgz#ae3bcb7e34c65879adfe77e19c30f86602b4bdb0" + integrity sha1-rjvLfjTGWHmt/nfhnDD4ZgK0vbA= + dependencies: + binary "^0.3.0" + graceful-fs "^4.1.3" + mkpath "^0.1.0" + nopt "^3.0.1" + q "^1.1.2" + readable-stream "^1.1.8" + touch "0.0.3" + +define-properties@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1" + integrity sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ== + dependencies: + object-keys "^1.0.12" + +define-property@^0.2.5: + version "0.2.5" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-0.2.5.tgz#c35b1ef918ec3c990f9a5bc57be04aacec5c8116" + integrity sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY= + dependencies: + is-descriptor "^0.1.0" + +define-property@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-1.0.0.tgz#769ebaaf3f4a63aad3af9e8d304c9bbe79bfb0e6" + integrity sha1-dp66rz9KY6rTr56NMEybvnm/sOY= + dependencies: + is-descriptor "^1.0.0" + +define-property@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-2.0.2.tgz#d459689e8d654ba77e02a817f8710d702cb16e9d" + integrity sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ== + dependencies: + is-descriptor "^1.0.2" + isobject "^3.0.1" + +defined@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" + integrity sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM= + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + +delegate@^3.1.2: + version "3.2.0" + resolved "https://registry.yarnpkg.com/delegate/-/delegate-3.2.0.tgz#b66b71c3158522e8ab5744f720d8ca0c2af59166" + integrity sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw== + +delegates@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" + integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= + +depd@~1.1.0, depd@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" + integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= + +destroy@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" + integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= + +detect-file@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/detect-file/-/detect-file-1.0.0.tgz#f0d66d03672a825cb1b73bdb3fe62310c8e552b7" + integrity sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc= + +"detective@>= 0.0.3": + version "5.2.0" + resolved "https://registry.yarnpkg.com/detective/-/detective-5.2.0.tgz#feb2a77e85b904ecdea459ad897cc90a99bd2a7b" + integrity sha512-6SsIx+nUUbuK0EthKjv0zrdnajCCXVYGmbYYiYjFVpzcjwEs/JMDZ8tPRG29J/HhN56t3GJp2cGSWDRjjot8Pg== + dependencies: + acorn-node "^1.6.1" + defined "^1.0.0" + minimist "^1.1.1" + +dom-serializer@0: + version "0.2.2" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51" + integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g== + dependencies: + domelementtype "^2.0.1" + entities "^2.0.0" + +domelementtype@1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f" + integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== + +domelementtype@^2.0.1: + version "2.1.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.1.0.tgz#a851c080a6d1c3d94344aed151d99f669edf585e" + integrity sha512-LsTgx/L5VpD+Q8lmsXSHW2WpA+eBlZ9HPf3erD1IoPF00/3JKHZ3BknUVA2QGDNu69ZNmyFmCWBSO45XjYKC5w== + +domhandler@2.3: + version "2.3.0" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.3.0.tgz#2de59a0822d5027fabff6f032c2b25a2a8abe738" + integrity sha1-LeWaCCLVAn+r/28DLCsloqir5zg= + dependencies: + domelementtype "1" + +domutils@1.5: + version "1.5.1" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" + integrity sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8= + dependencies: + dom-serializer "0" + domelementtype "1" + +duplexer@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" + integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== + +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + +ee-first@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" + integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= + +electron-to-chromium@^1.3.649: + version "1.3.653" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.653.tgz#1d98400eba330538a7fe169808c6bf9d83c44450" + integrity sha512-LehOhcl74u9fkV9Un6WahJ+Xh+0FZLCCDnKYis1Olx1DX2ugRww5PJicE65OG8yznMj8EOQZRcz6FSV1xKxqsA== + +encodeurl@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" + integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= + +engine.io-client@~3.1.0: + version "3.1.6" + resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-3.1.6.tgz#5bdeb130f8b94a50ac5cbeb72583e7a4a063ddfd" + integrity sha512-hnuHsFluXnsKOndS4Hv6SvUrgdYx1pk2NqfaDMW+GWdgfU3+/V25Cj7I8a0x92idSpa5PIhJRKxPvp9mnoLsfg== + dependencies: + component-emitter "1.2.1" + component-inherit "0.0.3" + debug "~3.1.0" + engine.io-parser "~2.1.1" + has-cors "1.1.0" + indexof "0.0.1" + parseqs "0.0.5" + parseuri "0.0.5" + ws "~3.3.1" + xmlhttprequest-ssl "~1.5.4" + yeast "0.1.2" + +engine.io-parser@~2.1.0, engine.io-parser@~2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-2.1.3.tgz#757ab970fbf2dfb32c7b74b033216d5739ef79a6" + integrity sha512-6HXPre2O4Houl7c4g7Ic/XzPnHBvaEmN90vtRO9uLmwtRqQmTOw0QMevL1TOfL2Cpu1VzsaTmMotQgMdkzGkVA== + dependencies: + after "0.8.2" + arraybuffer.slice "~0.0.7" + base64-arraybuffer "0.1.5" + blob "0.0.5" + has-binary2 "~1.0.2" + +engine.io@~3.1.0: + version "3.1.5" + resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-3.1.5.tgz#0e7ef9d690eb0b35597f1d4ad02a26ca2dba3845" + integrity sha512-D06ivJkYxyRrcEe0bTpNnBQNgP9d3xog+qZlLbui8EsMr/DouQpf5o9FzJnWYHEYE0YsFHllUv2R1dkgYZXHcA== + dependencies: + accepts "~1.3.4" + base64id "1.0.0" + cookie "0.3.1" + debug "~3.1.0" + engine.io-parser "~2.1.0" + ws "~3.3.1" + optionalDependencies: + uws "~9.14.0" + +entities@1.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-1.0.0.tgz#b2987aa3821347fcde642b24fdfc9e4fb712bf26" + integrity sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY= + +entities@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" + integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== + +eonasdan-bootstrap-datetimepicker@^4.17.47: + version "4.17.49" + resolved "https://registry.yarnpkg.com/eonasdan-bootstrap-datetimepicker/-/eonasdan-bootstrap-datetimepicker-4.17.49.tgz#5534ba581c1e7eb988dbf773e2fed8a7f48cc76a" + integrity sha512-7KZeDpkj+A6AtPR3XjX8gAnRPUkPSfW0OmMANG1dkUOPMtLSzbyoCjDIdEcfRtQPU5X0D9Gob7wWKn0h4QWy7A== + dependencies: + bootstrap "^3.3" + jquery "^3.5.1" + moment "^2.10" + moment-timezone "^0.4.0" + +errno@^0.1.1: + version "0.1.8" + resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f" + integrity sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A== + dependencies: + prr "~1.0.1" + +error@^7.0.0: + version "7.2.1" + resolved "https://registry.yarnpkg.com/error/-/error-7.2.1.tgz#eab21a4689b5f684fc83da84a0e390de82d94894" + integrity sha512-fo9HBvWnx3NGUKMvMwB/CBCMMrfEJgbDTVDEkPygA3Bdd3lM1OyCd+rbQ8BwnpF6GdVeOLDNmyL4N5Bg80ZvdA== + dependencies: + string-template "~0.2.1" + +escalade@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" + integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== + +escape-html@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" + integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg= + +escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5, escape-string-regexp@~1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= + +esprima@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" + integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== + +esri-leaflet@^2.3.0: + version "2.5.3" + resolved "https://registry.yarnpkg.com/esri-leaflet/-/esri-leaflet-2.5.3.tgz#ccf0c7463c7f9b6dfe27aab478740c3dbd8fe25b" + integrity sha512-zapunrhhhKyiVP5NCSfFjD7YqWYYYD3OONVjBFWZgX2KbD6ssUQ3KnXVo2U0hswWfJDIoHF7g9PLZ4rDNuQnvA== + dependencies: + "@terraformer/arcgis" "^2.0.7" + tiny-binary-search "^1.0.3" + +estree-walker@^0.5.0: + version "0.5.2" + resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.5.2.tgz#d3850be7529c9580d815600b53126515e146dd39" + integrity sha512-XpCnW/AE10ws/kDAs37cngSkvgIR8aN3G0MS85m7dUpuK2EREo9VJ00uvw6Dg/hXEpfsE1I1TvJOJr+Z+TL+ig== + +estree-walker@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362" + integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w== + +esutils@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== + +etag@~1.8.0: + version "1.8.1" + resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" + integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= + +eventemitter2@~0.4.13: + version "0.4.14" + resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-0.4.14.tgz#8f61b75cde012b2e9eb284d4545583b5643b61ab" + integrity sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas= + +exit@0.1.2, exit@0.1.x, exit@~0.1.1, exit@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" + integrity sha1-BjJjj42HfMghB9MKD/8aF8uhzQw= + +expand-brackets@^2.1.4: + version "2.1.4" + resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-2.1.4.tgz#b77735e315ce30f6b6eff0f83b04151a22449622" + integrity sha1-t3c14xXOMPa27/D4OwQVGiJEliI= + dependencies: + debug "^2.3.3" + define-property "^0.2.5" + extend-shallow "^2.0.1" + posix-character-classes "^0.1.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +expand-tilde@^2.0.0, expand-tilde@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/expand-tilde/-/expand-tilde-2.0.2.tgz#97e801aa052df02454de46b02bf621642cdc8502" + integrity sha1-l+gBqgUt8CRU3kawK/YhZCzchQI= + dependencies: + homedir-polyfill "^1.0.1" + +extend-shallow@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-2.0.1.tgz#51af7d614ad9a9f610ea1bafbb989d6b1c56890f" + integrity sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8= + dependencies: + is-extendable "^0.1.0" + +extend-shallow@^3.0.0, extend-shallow@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-3.0.2.tgz#26a71aaf073b39fb2127172746131c2704028db8" + integrity sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg= + dependencies: + assign-symbols "^1.0.0" + is-extendable "^1.0.1" + +extend@^3.0.0, extend@~3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +extglob@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" + integrity sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw== + dependencies: + array-unique "^0.3.2" + define-property "^1.0.0" + expand-brackets "^2.1.4" + extend-shallow "^2.0.1" + fragment-cache "^0.2.1" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= + +extsprintf@^1.2.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" + integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= + +fastsearch@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/fastsearch/-/fastsearch-1.11.1.tgz#2aeb3e9f6e9ef0e2edcbe809841bf4243fef6df1" + integrity sha1-Kus+n26e8OLty+gJhBv0JD/vbfE= + dependencies: + jquery ">=1.7" + +fastselect@^0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/fastselect/-/fastselect-0.7.3.tgz#b1434116ed0455ef9d881f646679dea6cb146fa7" + integrity sha1-sUNBFu0EVe+diB9kZnnepssUb6c= + dependencies: + fastsearch "^1.11.1" + jquery ">=1.7" + +faye-websocket@~0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.10.0.tgz#4e492f8d04dfb6f89003507f6edbf2d501e7c6f4" + integrity sha1-TkkvjQTftviQA1B/btvy1QHnxvQ= + dependencies: + websocket-driver ">=0.5.1" + +figures@^1.0.1: + version "1.7.0" + resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e" + integrity sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4= + dependencies: + escape-string-regexp "^1.0.5" + object-assign "^4.1.0" + +file-sync-cmp@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/file-sync-cmp/-/file-sync-cmp-0.1.1.tgz#a5e7a8ffbfa493b43b923bbd4ca89a53b63b612b" + integrity sha1-peeo/7+kk7Q7kju9TKiaU7Y7YSs= + +fill-range@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7" + integrity sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc= + dependencies: + extend-shallow "^2.0.1" + is-number "^3.0.0" + repeat-string "^1.6.1" + to-regex-range "^2.1.0" + +find-up@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73" + integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg== + dependencies: + locate-path "^3.0.0" + +findup-sync@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-2.0.0.tgz#9326b1488c22d1a6088650a86901b2d9a90a2cbc" + integrity sha1-kyaxSIwi0aYIhlCoaQGy2akKLLw= + dependencies: + detect-file "^1.0.0" + is-glob "^3.1.0" + micromatch "^3.0.4" + resolve-dir "^1.0.1" + +findup-sync@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" + integrity sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg== + dependencies: + detect-file "^1.0.0" + is-glob "^4.0.0" + micromatch "^3.0.4" + resolve-dir "^1.0.1" + +findup-sync@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-0.3.0.tgz#37930aa5d816b777c03445e1966cc6790a4c0b16" + integrity sha1-N5MKpdgWt3fANEXhlmzGeQpMCxY= + dependencies: + glob "~5.0.0" + +fined@^1.0.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/fined/-/fined-1.2.0.tgz#d00beccf1aa2b475d16d423b0238b713a2c4a37b" + integrity sha512-ZYDqPLGxDkDhDZBjZBb+oD1+j0rA4E0pXY50eplAAOPg2N/gUBSSk5IM1/QhPfyVo19lJ+CvXpqfvk+b2p/8Ng== + dependencies: + expand-tilde "^2.0.2" + is-plain-object "^2.0.3" + object.defaults "^1.1.0" + object.pick "^1.2.0" + parse-filepath "^1.0.1" + +flagged-respawn@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/flagged-respawn/-/flagged-respawn-1.0.1.tgz#e7de6f1279ddd9ca9aac8a5971d618606b3aab41" + integrity sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q== + +fontselect-jquery-plugin@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fontselect-jquery-plugin/-/fontselect-jquery-plugin-1.1.0.tgz#e4d53b9d237e270e3c7fcf77df5b0abd5e74e653" + integrity sha512-K8hh6ODJK0VNl+KhD5TWni4RXqolaxDY4sDbeowSSFa03FgpRz+zt8DD1/O+W6pDM3SGqB72WC9sI7H0StNtvA== + +for-in@^1.0.1, for-in@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" + integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= + +for-own@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b" + integrity sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs= + dependencies: + for-in "^1.0.1" + +foreachasync@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/foreachasync/-/foreachasync-3.0.0.tgz#5502987dc8714be3392097f32e0071c9dee07cf6" + integrity sha1-VQKYfchxS+M5IJfzLgBxyd7gfPY= + +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= + +form-data@~2.1.1: + version "2.1.4" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1" + integrity sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE= + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.5" + mime-types "^2.1.12" + +fragment-cache@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" + integrity sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk= + dependencies: + map-cache "^0.2.2" + +fresh@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.0.tgz#f474ca5e6a9246d6fd8e0953cfa9b9c805afa78e" + integrity sha1-9HTKXmqSRtb9jglTz6m5yAWvp44= + +fs-extra@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-3.0.1.tgz#3794f378c58b342ea7dbbb23095109c4b3b62291" + integrity sha1-N5TzeMWLNC6n27sjCVEJxLO2IpE= + dependencies: + graceful-fs "^4.1.2" + jsonfile "^3.0.0" + universalify "^0.1.0" + +fs-extra@4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-4.0.1.tgz#7fc0c6c8957f983f57f306a24e5b9ddd8d0dd880" + integrity sha1-f8DGyJV/mD9X8waiTlud3Y0N2IA= + dependencies: + graceful-fs "^4.1.2" + jsonfile "^3.0.0" + universalify "^0.1.0" + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + +gauge@~2.7.3: + version "2.7.4" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" + integrity sha1-LANAXHU4w51+s3sxcCLjJfsBi/c= + dependencies: + aproba "^1.0.3" + console-control-strings "^1.0.0" + has-unicode "^2.0.0" + object-assign "^4.1.0" + signal-exit "^3.0.0" + string-width "^1.0.1" + strip-ansi "^3.0.1" + wide-align "^1.1.0" + +gaze@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/gaze/-/gaze-1.1.2.tgz#847224677adb8870d679257ed3388fdb61e40105" + integrity sha1-hHIkZ3rbiHDWeSV+0ziP22HkAQU= + dependencies: + globule "^1.0.0" + +gaze@^1.1.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/gaze/-/gaze-1.1.3.tgz#c441733e13b927ac8c0ff0b4c3b033f28812924a" + integrity sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g== + dependencies: + globule "^1.0.0" + +gensync@^1.0.0-beta.1: + version "1.0.0-beta.2" + resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" + integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== + +get-down@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/get-down/-/get-down-1.2.0.tgz#6aeca5043ef0aeaa5170c90b0f746c650017a3c3" + integrity sha512-SbpdXn+fNmGN6pp/vBALveh3oCbWYTR3wb5qifauPSFhAfAVQ7TMOnAkVzQm4YZD2KABzJB3sA2kmMLaOylA2A== + dependencies: + decompress-zip "0.3.0" + graceful-fs "4.1.11" + junk "2.1.0" + mout "1.0.0" + q "1.5.0" + request "2.81.0" + request-progress "3.0.0" + retry "0.10.1" + tar "3.1.5" + tmp "0.0.31" + +get-intrinsic@^1.0.2: + version "1.1.1" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" + integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.1" + +get-value@^2.0.3, get-value@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" + integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= + +getobject@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/getobject/-/getobject-0.1.0.tgz#047a449789fa160d018f5486ed91320b6ec7885c" + integrity sha1-BHpEl4n6Fg0Bj1SG7ZEyC27HiFw= + +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= + dependencies: + assert-plus "^1.0.0" + +glob@7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + integrity sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +glob@^7.0.5, glob@^7.1.1, glob@^7.1.3, glob@~7.1.1, glob@~7.1.6: + version "7.1.6" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" + integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +glob@~5.0.0: + version "5.0.15" + resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" + integrity sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E= + dependencies: + inflight "^1.0.4" + inherits "2" + minimatch "2 || 3" + once "^1.3.0" + path-is-absolute "^1.0.0" + +global-modules@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea" + integrity sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg== + dependencies: + global-prefix "^1.0.1" + is-windows "^1.0.1" + resolve-dir "^1.0.0" + +global-prefix@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-1.0.2.tgz#dbf743c6c14992593c655568cb66ed32c0122ebe" + integrity sha1-2/dDxsFJklk8ZVVoy2btMsASLr4= + dependencies: + expand-tilde "^2.0.2" + homedir-polyfill "^1.0.1" + ini "^1.3.4" + is-windows "^1.0.1" + which "^1.2.14" + +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + +globule@^1.0.0: + version "1.3.2" + resolved "https://registry.yarnpkg.com/globule/-/globule-1.3.2.tgz#d8bdd9e9e4eef8f96e245999a5dee7eb5d8529c4" + integrity sha512-7IDTQTIu2xzXkT+6mlluidnWo+BypnbSoEVVQCGfzqnl5Ik8d3e1d4wycb8Rj9tWW+Z39uPWsdlquqiqPCd/pA== + dependencies: + glob "~7.1.1" + lodash "~4.17.10" + minimatch "~3.0.2" + +good-listener@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/good-listener/-/good-listener-1.2.2.tgz#d53b30cdf9313dffb7dc9a0d477096aa6d145c50" + integrity sha1-1TswzfkxPf+33JoNR3CWqm0UXFA= + dependencies: + delegate "^3.1.2" + +graceful-fs@4.1.11: + version "4.1.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" + integrity sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg= + +graceful-fs@^4.1.2, graceful-fs@^4.1.3, graceful-fs@^4.1.6, graceful-fs@^4.1.9: + version "4.2.4" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" + integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== + +grunt-babel@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/grunt-babel/-/grunt-babel-8.0.0.tgz#92ef63aafadf938c488dc2f926ac9846e0c93d1b" + integrity sha512-WuiZFvGzcyzlEoPIcY1snI234ydDWeWWV5bpnB7PZsOLHcDsxWKnrR1rMWEUsbdVPPjvIirwFNsuo4CbJmsdFQ== + +grunt-cli@^1.3.2, grunt-cli@~1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/grunt-cli/-/grunt-cli-1.3.2.tgz#60f12d12c1b5aae94ae3469c6b5fe24e960014e8" + integrity sha512-8OHDiZZkcptxVXtMfDxJvmN7MVJNE8L/yIcPb4HB7TlyFD1kDvjHrb62uhySsU14wJx9ORMnTuhRMQ40lH/orQ== + dependencies: + grunt-known-options "~1.1.0" + interpret "~1.1.0" + liftoff "~2.5.0" + nopt "~4.0.1" + v8flags "~3.1.1" + +grunt-contrib-clean@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/grunt-contrib-clean/-/grunt-contrib-clean-2.0.0.tgz#3be7ca480da4b740aa5e9d863e2f7e8b24f8a68b" + integrity sha512-g5ZD3ORk6gMa5ugZosLDQl3dZO7cI3R14U75hTM+dVLVxdMNJCPVmwf9OUt4v4eWgpKKWWoVK9DZc1amJp4nQw== + dependencies: + async "^2.6.1" + rimraf "^2.6.2" + +grunt-contrib-concat@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/grunt-contrib-concat/-/grunt-contrib-concat-1.0.1.tgz#61509863084e871d7e86de48c015259ed97745bd" + integrity sha1-YVCYYwhOhx1+ht5IwBUlntl3Rb0= + dependencies: + chalk "^1.0.0" + source-map "^0.5.3" + +grunt-contrib-copy@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/grunt-contrib-copy/-/grunt-contrib-copy-1.0.0.tgz#7060c6581e904b8ab0d00f076e0a8f6e3e7c3573" + integrity sha1-cGDGWB6QS4qw0A8HbgqPbj58NXM= + dependencies: + chalk "^1.1.1" + file-sync-cmp "^0.1.0" + +grunt-contrib-cssmin@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/grunt-contrib-cssmin/-/grunt-contrib-cssmin-3.0.0.tgz#1767c0a062f4b7353c5adcadc52dcff7197d4d89" + integrity sha512-eXpooYmVGKMs/xV7DzTLgJFPVOfMuawPD3x0JwhlH0mumq2NtH3xsxaHxp1Y3NKxp0j0tRhFS6kSBRsz6TuTGg== + dependencies: + chalk "^2.4.1" + clean-css "~4.2.1" + maxmin "^2.1.0" + +grunt-contrib-jshint@2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/grunt-contrib-jshint/-/grunt-contrib-jshint-2.1.0.tgz#3d789807579034299da1b41c4d70e1ba722973ed" + integrity sha512-65S2/C/6RfjY/umTxfwXXn+wVvaYmykHkHSsW6Q6rhkbv3oudTEgqnFFZvWzWCoHUb+3GMZLbP3oSrNyvshmIQ== + dependencies: + chalk "^2.4.2" + hooker "^0.2.3" + jshint "~2.10.2" + +grunt-contrib-less@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/grunt-contrib-less/-/grunt-contrib-less-2.0.0.tgz#ca4b78e15e54610cc04332d5fa0b196b203b14d9" + integrity sha512-nsaODoEMjVn61OuqPaFeFQpb4Qd/EbfxQDeYnh2oONXm8L5Gnuchtv59kl0V3hjiFdOkZlPILDc3ZrkoZI0PNw== + dependencies: + async "^2.0.0" + chalk "^1.0.0" + less "^3.0.4" + lodash "^4.17.10" + +grunt-contrib-uglify@4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/grunt-contrib-uglify/-/grunt-contrib-uglify-4.0.1.tgz#68a7b62fa045ce8e2c7574d1bdcd3b96b8a686b1" + integrity sha512-dwf8/+4uW1+7pH72WButOEnzErPGmtUvc8p08B0eQS/6ON0WdeQu0+WFeafaPTbbY1GqtS25lsHWaDeiTQNWPg== + dependencies: + chalk "^2.4.1" + maxmin "^2.1.0" + uglify-js "^3.5.0" + uri-path "^1.0.0" + +grunt-contrib-watch@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/grunt-contrib-watch/-/grunt-contrib-watch-1.1.0.tgz#c143ca5b824b288a024b856639a5345aedb78ed4" + integrity sha512-yGweN+0DW5yM+oo58fRu/XIRrPcn3r4tQx+nL7eMRwjpvk+rQY6R8o94BPK0i2UhTg9FN21hS+m8vR8v9vXfeg== + dependencies: + async "^2.6.0" + gaze "^1.1.0" + lodash "^4.17.10" + tiny-lr "^1.1.1" + +grunt-known-options@~1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/grunt-known-options/-/grunt-known-options-1.1.1.tgz#6cc088107bd0219dc5d3e57d91923f469059804d" + integrity sha512-cHwsLqoighpu7TuYj5RonnEuxGVFnztcUqTqp5rXFGYL4OuPFofwC4Ycg7n9fYwvK6F5WbYgeVOwph9Crs2fsQ== + +grunt-legacy-log-utils@~2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/grunt-legacy-log-utils/-/grunt-legacy-log-utils-2.1.0.tgz#49a8c7dc74051476dcc116c32faf9db8646856ef" + integrity sha512-lwquaPXJtKQk0rUM1IQAop5noEpwFqOXasVoedLeNzaibf/OPWjKYvvdqnEHNmU+0T0CaReAXIbGo747ZD+Aaw== + dependencies: + chalk "~4.1.0" + lodash "~4.17.19" + +grunt-legacy-log@~3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/grunt-legacy-log/-/grunt-legacy-log-3.0.0.tgz#1c6eaf92371ea415af31ea84ce50d434ef6d39c4" + integrity sha512-GHZQzZmhyq0u3hr7aHW4qUH0xDzwp2YXldLPZTCjlOeGscAOWWPftZG3XioW8MasGp+OBRIu39LFx14SLjXRcA== + dependencies: + colors "~1.1.2" + grunt-legacy-log-utils "~2.1.0" + hooker "~0.2.3" + lodash "~4.17.19" + +grunt-legacy-util@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/grunt-legacy-util/-/grunt-legacy-util-2.0.0.tgz#34d20f2a26c6adebfe9a9bdc8823f7016b0369c3" + integrity sha512-ZEmYFB44bblwPE2oz3q3ygfF6hseQja9tx8I3UZIwbUik32FMWewA+d1qSFicMFB+8dNXDkh35HcDCWlpRsGlA== + dependencies: + async "~1.5.2" + exit "~0.1.1" + getobject "~0.1.0" + hooker "~0.2.3" + lodash "~4.17.20" + underscore.string "~3.3.5" + which "~1.3.0" + +grunt-text-replace@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/grunt-text-replace/-/grunt-text-replace-0.4.0.tgz#db9d9ce59e2fe49da277e9dbc195c3e11cfb16c2" + integrity sha1-252c5Z4v5J2id+nbwZXD4Rz7FsI= + +grunt@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/grunt/-/grunt-1.3.0.tgz#55db6ccd80c6fb53722e496f680620a2e681f809" + integrity sha512-6ILlMXv11/4cxuhSMfSU+SfvbxrPuqZrAtLN64+tZpQ3DAKfSQPQHRbTjSbdtxfyQhGZPtN0bDZJ/LdCM5WXXA== + dependencies: + dateformat "~3.0.3" + eventemitter2 "~0.4.13" + exit "~0.1.2" + findup-sync "~0.3.0" + glob "~7.1.6" + grunt-cli "~1.3.2" + grunt-known-options "~1.1.0" + grunt-legacy-log "~3.0.0" + grunt-legacy-util "~2.0.0" + iconv-lite "~0.4.13" + js-yaml "~3.14.0" + minimatch "~3.0.4" + mkdirp "~1.0.4" + nopt "~3.0.6" + rimraf "~3.0.2" + +gzip-size@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-3.0.0.tgz#546188e9bdc337f673772f81660464b389dce520" + integrity sha1-VGGI6b3DN/Zzdy+BZgRks4nc5SA= + dependencies: + duplexer "^0.1.1" + +handlebars@4.0.10: + version "4.0.10" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.10.tgz#3d30c718b09a3d96f23ea4cc1f403c4d3ba9ff4f" + integrity sha1-PTDHGLCaPZbyPqTMH0A8TTup/08= + dependencies: + async "^1.4.0" + optimist "^0.6.1" + source-map "^0.4.4" + optionalDependencies: + uglify-js "^2.6" + +handlebars@^4.1.2: + version "4.7.6" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.6.tgz#d4c05c1baf90e9945f77aa68a7a219aa4a7df74e" + integrity sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA== + dependencies: + minimist "^1.2.5" + neo-async "^2.6.0" + source-map "^0.6.1" + wordwrap "^1.0.0" + optionalDependencies: + uglify-js "^3.1.4" + +har-schema@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" + integrity sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4= + +har-validator@~4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-4.2.1.tgz#33481d0f1bbff600dd203d75812a6a5fba002e2a" + integrity sha1-M0gdDxu/9gDdID11gSpqX7oALio= + dependencies: + ajv "^4.9.1" + har-schema "^1.0.5" + +has-ansi@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" + integrity sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE= + dependencies: + ansi-regex "^2.0.0" + +has-binary2@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-binary2/-/has-binary2-1.0.3.tgz#7776ac627f3ea77250cfc332dab7ddf5e4f5d11d" + integrity sha512-G1LWKhDSvhGeAQ8mPVQlqNcOB2sJdwATtZKl2pDKKHfpf/rYj24lkinxf69blJbnsvtqqNU+L3SL50vzZhXOnw== + dependencies: + isarray "2.0.1" + +has-color@~0.1.0: + version "0.1.7" + resolved "https://registry.yarnpkg.com/has-color/-/has-color-0.1.7.tgz#67144a5260c34fc3cca677d041daf52fe7b78b2f" + integrity sha1-ZxRKUmDDT8PMpnfQQdr1L+e3iy8= + +has-cors@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/has-cors/-/has-cors-1.1.0.tgz#5e474793f7ea9843d1bb99c23eef49ff126fff39" + integrity sha1-XkdHk/fqmEPRu5nCPu9J/xJv/zk= + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= + +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +has-symbols@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.1.tgz#9f5214758a44196c406d9bd76cebf81ec2dd31e8" + integrity sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg== + +has-unicode@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" + integrity sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk= + +has-value@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/has-value/-/has-value-0.3.1.tgz#7b1f58bada62ca827ec0a2078025654845995e1f" + integrity sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8= + dependencies: + get-value "^2.0.3" + has-values "^0.1.4" + isobject "^2.0.0" + +has-value@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-value/-/has-value-1.0.0.tgz#18b281da585b1c5c51def24c930ed29a0be6b177" + integrity sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc= + dependencies: + get-value "^2.0.6" + has-values "^1.0.0" + isobject "^3.0.0" + +has-values@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/has-values/-/has-values-0.1.4.tgz#6d61de95d91dfca9b9a02089ad384bff8f62b771" + integrity sha1-bWHeldkd/Km5oCCJrThL/49it3E= + +has-values@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-values/-/has-values-1.0.0.tgz#95b0b63fec2146619a6fe57fe75628d5a39efe4f" + integrity sha1-lbC2P+whRmGab+V/51Yo1aOe/k8= + dependencies: + is-number "^3.0.0" + kind-of "^4.0.0" + +has@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + +hawk@~3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" + integrity sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ= + dependencies: + boom "2.x.x" + cryptiles "2.x.x" + hoek "2.x.x" + sntp "1.x.x" + +hoek@2.x.x: + version "2.16.3" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" + integrity sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0= + +homedir-polyfill@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8" + integrity sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA== + dependencies: + parse-passwd "^1.0.0" + +hooker@^0.2.3, hooker@~0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/hooker/-/hooker-0.2.3.tgz#b834f723cc4a242aa65963459df6d984c5d3d959" + integrity sha1-uDT3I8xKJCqmWWNFnfbZhMXT2Vk= + +htmlparser2@3.8.x: + version "3.8.3" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.8.3.tgz#996c28b191516a8be86501a7d79757e5c70c1068" + integrity sha1-mWwosZFRaovoZQGn15dX5ccMEGg= + dependencies: + domelementtype "1" + domhandler "2.3" + domutils "1.5" + entities "1.0" + readable-stream "1.1" + +http-errors@~1.6.1: + version "1.6.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.3.tgz#8b55680bb4be283a0b5bf4ea2e38580be1d9320d" + integrity sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0= + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.0" + statuses ">= 1.4.0 < 2" + +http-parser-js@>=0.5.1: + version "0.5.3" + resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.5.3.tgz#01d2709c79d41698bb01d4decc5e9da4e4a033d9" + integrity sha512-t7hjvef/5HEK7RWTdUzVUhl8zkEu+LlaE0IYzdMuvbSDipxBRpOn4Uhw8ZyECEa808iVT8XCjzo6xmYt4CiLZg== + +http-signature@~1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf" + integrity sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8= + dependencies: + assert-plus "^0.2.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + +hyperscript@~1.0.1: + version "1.0.7" + resolved "https://registry.yarnpkg.com/hyperscript/-/hyperscript-1.0.7.tgz#d88ed3c51934c6d6fa35b2abfd7c00f100820f65" + integrity sha1-2I7TxRk0xtb6NbKr/XwA8QCCD2U= + dependencies: + browser-split "0.0.0" + class-list "~0.1.0" + +iconv-lite@~0.4.13: + version "0.4.24" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== + dependencies: + safer-buffer ">= 2.1.2 < 3" + +ieee754@^1.1.6: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + +image-size@~0.5.0: + version "0.5.5" + resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c" + integrity sha1-Cd/Uq50g4p6xw+gLiZA3jfnjy5w= + +indexof@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/indexof/-/indexof-0.0.1.tgz#82dc336d232b9062179d05ab3293a66059fd435d" + integrity sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10= + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2, inherits@~2.0.1, inherits@~2.0.3: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +inherits@2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= + +ini@^1.3.4: + version "1.3.8" + resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" + integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== + +interpret@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.1.0.tgz#7ed1b1410c6a0e0f78cf95d3b8440c63f78b8614" + integrity sha1-ftGxQQxqDg94z5XTuEQMY/eLhhQ= + +intersect@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/intersect/-/intersect-0.1.0.tgz#01a651345bd55a7be50ae2f0f7257f8ba1142ccb" + integrity sha1-AaZRNFvVWnvlCuLw9yV/i6EULMs= + +is-absolute@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-absolute/-/is-absolute-1.0.0.tgz#395e1ae84b11f26ad1795e73c17378e48a301576" + integrity sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA== + dependencies: + is-relative "^1.0.0" + is-windows "^1.0.1" + +is-accessor-descriptor@^0.1.6: + version "0.1.6" + resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" + integrity sha1-qeEss66Nh2cn7u84Q/igiXtcmNY= + dependencies: + kind-of "^3.0.2" + +is-accessor-descriptor@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz#169c2f6d3df1f992618072365c9b0ea1f6878656" + integrity sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ== + dependencies: + kind-of "^6.0.0" + +is-buffer@^1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + +is-core-module@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.2.0.tgz#97037ef3d52224d85163f5597b2b63d9afed981a" + integrity sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ== + dependencies: + has "^1.0.3" + +is-data-descriptor@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" + integrity sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y= + dependencies: + kind-of "^3.0.2" + +is-data-descriptor@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz#d84876321d0e7add03990406abbbbd36ba9268c7" + integrity sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ== + dependencies: + kind-of "^6.0.0" + +is-descriptor@^0.1.0: + version "0.1.6" + resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-0.1.6.tgz#366d8240dde487ca51823b1ab9f07a10a78251ca" + integrity sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg== + dependencies: + is-accessor-descriptor "^0.1.6" + is-data-descriptor "^0.1.4" + kind-of "^5.0.0" + +is-descriptor@^1.0.0, is-descriptor@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-1.0.2.tgz#3b159746a66604b04f8c81524ba365c5f14d86ec" + integrity sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg== + dependencies: + is-accessor-descriptor "^1.0.0" + is-data-descriptor "^1.0.0" + kind-of "^6.0.2" + +is-extendable@^0.1.0, is-extendable@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" + integrity sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik= + +is-extendable@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-1.0.1.tgz#a7470f9e426733d81bd81e1155264e3a3507cab4" + integrity sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA== + dependencies: + is-plain-object "^2.0.4" + +is-extglob@^2.1.0, is-extglob@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= + +is-fullwidth-code-point@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" + integrity sha1-754xOG8DGn8NZDr4L95QxFfvAMs= + dependencies: + number-is-nan "^1.0.0" + +is-fullwidth-code-point@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" + integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8= + +is-glob@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-3.1.0.tgz#7ba5ae24217804ac70707b96922567486cc3e84a" + integrity sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo= + dependencies: + is-extglob "^2.1.0" + +is-glob@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" + integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg== + dependencies: + is-extglob "^2.1.1" + +is-module@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591" + integrity sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE= + +is-number@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" + integrity sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU= + dependencies: + kind-of "^3.0.2" + +is-plain-object@^2.0.3, is-plain-object@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" + integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== + dependencies: + isobject "^3.0.1" + +is-relative@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-relative/-/is-relative-1.0.0.tgz#a1bb6935ce8c5dba1e8b9754b9b2dcc020e2260d" + integrity sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA== + dependencies: + is-unc-path "^1.0.0" + +is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= + +is-unc-path@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-unc-path/-/is-unc-path-1.0.0.tgz#d731e8898ed090a12c352ad2eaed5095ad322c9d" + integrity sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ== + dependencies: + unc-path-regex "^0.1.2" + +is-what@^3.7.1: + version "3.12.0" + resolved "https://registry.yarnpkg.com/is-what/-/is-what-3.12.0.tgz#f4405ce4bd6dd420d3ced51a026fb90e03705e55" + integrity sha512-2ilQz5/f/o9V7WRWJQmpFYNmQFZ9iM+OXRonZKcYgTkCzjb949Vi4h282PD1UfmgHk666rcWonbRJ++KI41VGw== + +is-windows@^1.0.1, is-windows@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" + integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA== + +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= + +isarray@1.0.0, isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +isarray@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.1.tgz#a37d94ed9cda2d59865c9f76fe596ee1f338741e" + integrity sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4= + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= + +isobject@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" + integrity sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk= + dependencies: + isarray "1.0.0" + +isobject@^3.0.0, isobject@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" + integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= + +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= + +"joose@>= 3.14.0", "joose@>= 3.16.0": + version "3.50.0" + resolved "https://registry.yarnpkg.com/joose/-/joose-3.50.0.tgz#0926824d43f640a0f47370bfca748a929344b483" + integrity sha1-CSaCTUP2QKD0c3C/ynSKkpNEtIM= + dependencies: + detective ">= 0.0.3" + optimist "*" + temp "*" + +"joosex-namespace-depended@>= 0.11.0": + version "0.18.0" + resolved "https://registry.yarnpkg.com/joosex-namespace-depended/-/joosex-namespace-depended-0.18.0.tgz#3d82c113a331b2b1b860193a7bf3fc3ed56f9f1c" + integrity sha1-PYLBE6MxsrG4YBk6e/P8PtVvnxw= + dependencies: + joose ">= 3.16.0" + joosex-simplerequest ">= 0.2.1" + +"joosex-simplerequest@>= 0.2.1": + version "0.2.2" + resolved "https://registry.yarnpkg.com/joosex-simplerequest/-/joosex-simplerequest-0.2.2.tgz#0c9e1cb0d00a2e760abc1fded3be4977f81847c0" + integrity sha1-DJ4csNAKLnYKvB/e075Jd/gYR8A= + dependencies: + joose ">= 3.14.0" + +jq-ajax-progress@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/jq-ajax-progress/-/jq-ajax-progress-1.0.0.tgz#8cc055e4880d3929478ec67a71ea3db42a6bae81" + integrity sha512-qaC2Ixo4N56aqDA/k3RB5WsTLwMhw2muFTqOk86z11fzlkMaeJw/oTDxKhyF68D6WcfEL5CHat9SSUF79uYXpg== + +jquery-browserify@~1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/jquery-browserify/-/jquery-browserify-1.8.1.tgz#0dea9d741a4f4115ae1980c4531b22502991a015" + integrity sha1-DeqddBpPQRWuGYDEUxsiUCmRoBU= + +jquery-ui-dist@^1.12.1: + version "1.12.1" + resolved "https://registry.yarnpkg.com/jquery-ui-dist/-/jquery-ui-dist-1.12.1.tgz#5c0815d3cc6f90ff5faaf5b268a6e23b4ca904fa" + integrity sha1-XAgV08xvkP9fqvWyaKbiO0ypBPo= + +jquery-waypoints@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/jquery-waypoints/-/jquery-waypoints-2.0.5.tgz#c302b188e2957ed763098793304a76356aeeee04" + integrity sha1-wwKxiOKVftdjCYeTMEp2NWru7gQ= + +jquery@3.5.1, "jquery@>=1.5.0 <4.0", jquery@>=1.7, jquery@^3.3.1, jquery@^3.4.1, jquery@^3.5.1: + version "3.5.1" + resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.5.1.tgz#d7b4d08e1bfdb86ad2f1a3d039ea17304717abb5" + integrity sha512-XwIBPqcMn57FxfT+Go5pzySnm4KWkT1Tv7gjrpT1srtf8Weynl6R273VJ5GjkRb51IzMp5nbaPjJXMWeju2MKg== + +jquery@^2.1.0: + version "2.2.4" + resolved "https://registry.yarnpkg.com/jquery/-/jquery-2.2.4.tgz#2c89d6889b5eac522a7eea32c14521559c6cbf02" + integrity sha1-LInWiJterFIqfuoywUUhVZxsvwI= + +jquery@~2.1.3: + version "2.1.4" + resolved "https://registry.yarnpkg.com/jquery/-/jquery-2.1.4.tgz#228bde698a0c61431dc2630a6a154f15890d2317" + integrity sha1-IoveaYoMYUMdwmMKahVPFYkNIxc= + +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + +js-yaml@~3.14.0: + version "3.14.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +js2xmlparser@~3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/js2xmlparser/-/js2xmlparser-3.0.0.tgz#3fb60eaa089c5440f9319f51760ccd07e2499733" + integrity sha1-P7YOqgicVED5MZ9RdgzNB+JJlzM= + dependencies: + xmlcreate "^1.0.1" + +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= + +jsdoc@3.5.4: + version "3.5.4" + resolved "https://registry.yarnpkg.com/jsdoc/-/jsdoc-3.5.4.tgz#ceeef7c4bac4335cb10ff41e3a0f58939a534428" + integrity sha512-VmTw0J+2L16IxAe0JSDSAcH0F+DbZxaj8wN1AjHtKMQU/hO0ciIl5ZE93XqrrFIbknobuqHKJCXZj6+Hk57MjA== + dependencies: + babylon "~7.0.0-beta.19" + bluebird "~3.5.0" + catharsis "~0.8.9" + escape-string-regexp "~1.0.5" + js2xmlparser "~3.0.0" + klaw "~2.0.0" + marked "~0.3.6" + mkdirp "~0.5.1" + requizzle "~0.2.1" + strip-json-comments "~2.0.1" + taffydb "2.6.2" + underscore "~1.8.3" + +jsesc@^2.5.1: + version "2.5.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== + +jsesc@~0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" + integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= + +jshint@~2.10.2: + version "2.10.3" + resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.10.3.tgz#98dc765bf6920b41bc2719f76b8739d6f6e93a9c" + integrity sha512-d8AoXcNNYzmm7cdmulQ3dQApbrPYArtVBO6n4xOICe4QsXGNHCAKDcFORzqP52LhK61KX0VhY39yYzCsNq+bxQ== + dependencies: + cli "~1.0.0" + console-browserify "1.1.x" + exit "0.1.x" + htmlparser2 "3.8.x" + lodash "~4.17.11" + minimatch "~3.0.2" + shelljs "0.3.x" + strip-json-comments "1.0.x" + +json-schema@0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" + integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= + +json-stable-stringify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af" + integrity sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8= + dependencies: + jsonify "~0.0.0" + +json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= + +json2@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/json2/-/json2-0.4.0.tgz#4f658ea900acc5acd7f8565977f16f41287810f9" + integrity sha1-T2WOqQCsxazX+FZZd/FvQSh4EPk= + dependencies: + joose ">= 3.14.0" + joosex-namespace-depended ">= 0.11.0" + +json5@^2.1.2: + version "2.2.0" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" + integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA== + dependencies: + minimist "^1.2.5" + +jsonfile@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-3.0.1.tgz#a5ecc6f65f53f662c4415c7675a0331d0992ec66" + integrity sha1-pezG9l9T9mLEQVx2daAzHQmS7GY= + optionalDependencies: + graceful-fs "^4.1.6" + +jsonify@~0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" + integrity sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM= + +jsprim@^1.2.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" + integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.2.3" + verror "1.10.0" + +junk@2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/junk/-/junk-2.1.0.tgz#f431b4b7f072dc500a5f10ce7f4ec71930e70134" + integrity sha1-9DG0t/By3FAKXxDOf07HGTDnATQ= + +kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ= + dependencies: + is-buffer "^1.1.5" + +kind-of@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-4.0.0.tgz#20813df3d712928b207378691a45066fae72dd57" + integrity sha1-IIE989cSkosgc3hpGkUGb65y3Vc= + dependencies: + is-buffer "^1.1.5" + +kind-of@^5.0.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d" + integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw== + +kind-of@^6.0.0, kind-of@^6.0.2: + version "6.0.3" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" + integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== + +klaw@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/klaw/-/klaw-2.0.0.tgz#59c128e0dc5ce410201151194eeb9cbf858650f6" + integrity sha1-WcEo4Nxc5BAgEVEZTuucv4WGUPY= + dependencies: + graceful-fs "^4.1.9" + +lazy-cache@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e" + integrity sha1-odePw6UEdMuAhF07O24dpJpEbo4= + +leaflet-easybutton@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/leaflet-easybutton/-/leaflet-easybutton-2.4.0.tgz#3ad8ad31c9cf6d5ca7a6a1988f82eed13ff2c1af" + integrity sha512-O+qsQq4zTF6ds8VClnytobTH/MKalctlPpiA8L+bNKHP14J3lgJpvEd/jSpq9mHTI6qOzRAvbQX6wS6qNwThvg== + dependencies: + leaflet "^1.0.1" + +leaflet-fullscreen@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/leaflet-fullscreen/-/leaflet-fullscreen-1.0.2.tgz#09c61c4bac45f63b2ee126afd87e5cd97650fc1b" + integrity sha1-CcYcS6xF9jsu4Sav2H5c2XZQ/Bs= + +leaflet-measure@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/leaflet-measure/-/leaflet-measure-3.1.0.tgz#97777c206a1f8e1ee5a260ffb4128db18fce9cd0" + integrity sha512-ln5c9UNaWDEd24tIzDt9hwnpb8OaCPBfSWNBg2H8rb9SA3cbXW9+NqohA6/8TdsNDGJZr36woXMrqRq07Pcl3w== + dependencies: + "@turf/area" "^5.1.5" + "@turf/length" "^5.1.5" + lodash "^4.17.5" + +leaflet-navbar@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/leaflet-navbar/-/leaflet-navbar-1.0.0.tgz#3af2a04344a21a6f3f440a98886c0a9e6676224e" + integrity sha512-cRFfD/V1dRxFxVB377DAi12c29krcLIbbz3Tu4Jk/nlvU+a0M5L2dr5zvMHVzlq8ioJ0MxvDOPl1YEP8STh2bQ== + +leaflet-transparency@^0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/leaflet-transparency/-/leaflet-transparency-0.0.5.tgz#2fc42f1efa9e6edbb3dd50454853220466646f21" + integrity sha512-Cn28k/Zr8BSBJSXmIiZk/lRmQX56yqtL3Kixz+kD1Rywpg/MISfEfBZZuELB6jGqvip9+gs7ZWjy//RLHlAjwQ== + +leaflet.control.opacity@^1.3.0: + version "1.4.2" + resolved "https://registry.yarnpkg.com/leaflet.control.opacity/-/leaflet.control.opacity-1.4.2.tgz#c5971f419110a60ef3ae1e30be6e29c676c851e0" + integrity sha512-03HpXVLA/Wq8s8pSmR7IxGMGQwS4U6a8LNbMpgmO/ZEUudWvfZZVQCCIDB5bH+y0Pk3eAD4/TS2i8MMxNQDOPg== + +leaflet.locatecontrol@^0.67.0: + version "0.67.0" + resolved "https://registry.yarnpkg.com/leaflet.locatecontrol/-/leaflet.locatecontrol-0.67.0.tgz#adfeebfd662f61d639a5a5fba362368579f8ac55" + integrity sha512-3R8jYBwYyXc5Ka7bZvx6naWUvfmJTGJfZ2o/ZWombhCK7NkOxEpwFA3bhW9/BRZ99cLyjeNuLUaLwcPrMKYrjA== + +leaflet.pancontrol@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/leaflet.pancontrol/-/leaflet.pancontrol-1.0.0.tgz#25f31639ad55b40de27df7dee34de5c607cfa71b" + integrity sha1-JfMWOa1VtA3ifffe403lxgfPpxs= + dependencies: + leaflet "~1.0.2" + +leaflet@0.7.x: + version "0.7.7" + resolved "https://registry.yarnpkg.com/leaflet/-/leaflet-0.7.7.tgz#1e352ba54e63d076451fa363c900890cb2cf75ee" + integrity sha1-HjUrpU5j0HZFH6NjyQCJDLLPde4= + +leaflet@^1.0.1, leaflet@^1.5.1: + version "1.7.1" + resolved "https://registry.yarnpkg.com/leaflet/-/leaflet-1.7.1.tgz#10d684916edfe1bf41d688a3b97127c0322a2a19" + integrity sha512-/xwPEBidtg69Q3HlqPdU3DnrXQOvQU/CCHA1tcDQVzOwm91YMYaILjNp7L4Eaw5Z4sOYdbBz6koWyibppd8Zqw== + +leaflet@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/leaflet/-/leaflet-1.0.3.tgz#1f401b98b45c8192134c6c8d69686253805007c8" + integrity sha1-H0AbmLRcgZITTGyNaWhiU4BQB8g= + +less@^3.0.4: + version "3.13.1" + resolved "https://registry.yarnpkg.com/less/-/less-3.13.1.tgz#0ebc91d2a0e9c0c6735b83d496b0ab0583077909" + integrity sha512-SwA1aQXGUvp+P5XdZslUOhhLnClSLIjWvJhmd+Vgib5BFIr9lMNlQwmwUNOjXThF/A0x+MCYYPeWEfeWiLRnTw== + dependencies: + copy-anything "^2.0.1" + tslib "^1.10.0" + optionalDependencies: + errno "^0.1.1" + graceful-fs "^4.1.2" + image-size "~0.5.0" + make-dir "^2.1.0" + mime "^1.4.1" + native-request "^1.0.5" + source-map "~0.6.0" + +liftoff@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/liftoff/-/liftoff-3.1.0.tgz#c9ba6081f908670607ee79062d700df062c52ed3" + integrity sha512-DlIPlJUkCV0Ips2zf2pJP0unEoT1kwYhiiPUGF3s/jtxTCjziNLoiVVh+jqWOWeFi6mmwQ5fNxvAUyPad4Dfog== + dependencies: + extend "^3.0.0" + findup-sync "^3.0.0" + fined "^1.0.1" + flagged-respawn "^1.0.0" + is-plain-object "^2.0.4" + object.map "^1.0.0" + rechoir "^0.6.2" + resolve "^1.1.7" + +liftoff@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/liftoff/-/liftoff-2.5.0.tgz#2009291bb31cea861bbf10a7c15a28caf75c31ec" + integrity sha1-IAkpG7Mc6oYbvxCnwVooyvdcMew= + dependencies: + extend "^3.0.0" + findup-sync "^2.0.0" + fined "^1.0.1" + flagged-respawn "^1.0.0" + is-plain-object "^2.0.4" + object.map "^1.0.0" + rechoir "^0.6.2" + resolve "^1.1.7" + +list.js@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/list.js/-/list.js-1.5.0.tgz#a4cbfc8281ddefc02fdb2d30c8748bfae25fbcda" + integrity sha1-pMv8goHd78Av2y0wyHSL+uJfvNo= + dependencies: + string-natural-compare "^2.0.2" + +list@^2.0.19: + version "2.0.19" + resolved "https://registry.yarnpkg.com/list/-/list-2.0.19.tgz#370a3d7d3e24cfd5ced2c89cda2baf28e31e2830" + integrity sha512-nnVaRp4RaMAQkCpypTThsdxKqgPMiSwJq93eAm2/IbpUa8sd04XKBhkKu+bMk63HmdjK8b8Cuh4xARHWX2ye/Q== + +livereload-js@^2.3.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.4.0.tgz#447c31cf1ea9ab52fc20db615c5ddf678f78009c" + integrity sha512-XPQH8Z2GDP/Hwz2PCDrh2mth4yFejwA1OZ/81Ti3LgKyhDcEjsSsqFWZojHG0va/duGd+WyosY7eXLDoOyqcPw== + +load-grunt-tasks@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/load-grunt-tasks/-/load-grunt-tasks-5.1.0.tgz#14894c27a7e34ebbef9937c39cc35c573cd04c1c" + integrity sha512-oNj0Jlka1TsfDe+9He0kcA1cRln+TMoTsEByW7ij6kyktNLxBKJtslCFEvFrLC2Dj0S19IWJh3fOCIjLby2Xrg== + dependencies: + arrify "^2.0.1" + multimatch "^4.0.0" + pkg-up "^3.1.0" + resolve-pkg "^2.0.0" + +locate-path@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e" + integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A== + dependencies: + p-locate "^3.0.0" + path-exists "^3.0.0" + +lodash@4.17.4: + version "4.17.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" + integrity sha1-eCA6TRwyiuHYbcpkYONptX9AVa4= + +lodash@^4.14.0, lodash@^4.17.10, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.5, lodash@~4.17.10, lodash@~4.17.11, lodash@~4.17.19, lodash@~4.17.20: + version "4.17.20" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" + integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== + +longest@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" + integrity sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc= + +magic-string@*: + version "0.25.7" + resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051" + integrity sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA== + dependencies: + sourcemap-codec "^1.4.4" + +magic-string@^0.22.4: + version "0.22.5" + resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.22.5.tgz#8e9cf5afddf44385c1da5bc2a6a0dbd10b03657e" + integrity sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w== + dependencies: + vlq "^0.2.2" + +make-dir@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5" + integrity sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA== + dependencies: + pify "^4.0.1" + semver "^5.6.0" + +make-iterator@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/make-iterator/-/make-iterator-1.0.1.tgz#29b33f312aa8f547c4a5e490f56afcec99133ad6" + integrity sha512-pxiuXh0iVEq7VM7KMIhs5gxsfxCux2URptUQaXo4iZZJxBAzTPOLE2BumO5dbfVYq/hBJFBR/a1mFDmOx5AGmw== + dependencies: + kind-of "^6.0.2" + +map-cache@^0.2.0, map-cache@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf" + integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8= + +map-visit@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f" + integrity sha1-7Nyo8TFE5mDxtb1B8S80edmN+48= + dependencies: + object-visit "^1.0.0" + +marked@~0.3.6: + version "0.3.19" + resolved "https://registry.yarnpkg.com/marked/-/marked-0.3.19.tgz#5d47f709c4c9fc3c216b6d46127280f40b39d790" + integrity sha512-ea2eGWOqNxPcXv8dyERdSr/6FmzvWwzjMxpfGB/sbMccXoct+xY+YukPD+QTUZwyvK7BZwcr4m21WBOW41pAkg== + +maxmin@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/maxmin/-/maxmin-2.1.0.tgz#4d3b220903d95eee7eb7ac7fa864e72dc09a3166" + integrity sha1-TTsiCQPZXu5+t6x/qGTnLcCaMWY= + dependencies: + chalk "^1.0.0" + figures "^1.0.1" + gzip-size "^3.0.0" + pretty-bytes "^3.0.0" + +mgrs@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/mgrs/-/mgrs-1.0.0.tgz#fb91588e78c90025672395cb40b25f7cd6ad1829" + integrity sha1-+5FYjnjJACVnI5XLQLJffNatGCk= + +micromatch@^3.0.4: + version "3.1.10" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23" + integrity sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg== + dependencies: + arr-diff "^4.0.0" + array-unique "^0.3.2" + braces "^2.3.1" + define-property "^2.0.2" + extend-shallow "^3.0.2" + extglob "^2.0.4" + fragment-cache "^0.2.1" + kind-of "^6.0.2" + nanomatch "^1.2.9" + object.pick "^1.3.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.2" + +mime-db@1.45.0: + version "1.45.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.45.0.tgz#cceeda21ccd7c3a745eba2decd55d4b73e7879ea" + integrity sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w== + +mime-types@^2.1.12, mime-types@~2.1.24, mime-types@~2.1.7: + version "2.1.28" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.28.tgz#1160c4757eab2c5363888e005273ecf79d2a0ecd" + integrity sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ== + dependencies: + mime-db "1.45.0" + +mime@1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" + integrity sha1-EV+eO2s9rylZmDyzjxSaLUDrXVM= + +mime@^1.4.1: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + +mime@^2.4.4: + version "2.5.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.0.tgz#2b4af934401779806ee98026bb42e8c1ae1876b1" + integrity sha512-ft3WayFSFUVBuJj7BMLKAQcSlItKtfjsKDDsii3rqFDAZ7t11zRe8ASw/GlmivGwVUYtwkQrxiGGpL6gFvB0ag== + +"minimatch@2 || 3", minimatch@3.0.4, minimatch@^3.0.4, minimatch@~3.0.2, minimatch@~3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + dependencies: + brace-expansion "^1.1.7" + +minimist@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0= + +minimist@^1.1.1, minimist@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" + integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + integrity sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8= + +minipass@^2.0.2, minipass@^2.9.0: + version "2.9.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6" + integrity sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg== + dependencies: + safe-buffer "^5.1.2" + yallist "^3.0.0" + +minizlib@^1.0.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d" + integrity sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q== + dependencies: + minipass "^2.9.0" + +mixin-deep@^1.2.0: + version "1.3.2" + resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566" + integrity sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA== + dependencies: + for-in "^1.0.2" + is-extendable "^1.0.1" + +mkdirp@0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM= + dependencies: + minimist "0.0.8" + +mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1: + version "0.5.5" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" + integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== + dependencies: + minimist "^1.2.5" + +mkdirp@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + +mkpath@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/mkpath/-/mkpath-0.1.0.tgz#7554a6f8d871834cc97b5462b122c4c124d6de91" + integrity sha1-dVSm+Nhxg0zJe1RisSLEwSTW3pE= + +moment-timezone@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.4.1.tgz#81f598c3ad5e22cdad796b67ecd8d88d0f5baa06" + integrity sha1-gfWYw61eIs2teWtn7NjYjQ9bqgY= + dependencies: + moment ">= 2.6.0" + +moment-timezone@^0.5.26: + version "0.5.32" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.32.tgz#db7677cc3cc680fd30303ebd90b0da1ca0dfecc2" + integrity sha512-Z8QNyuQHQAmWucp8Knmgei8YNo28aLjJq6Ma+jy1ZSpSk5nyfRT8xgUbSQvD2+2UajISfenndwvFuH3NGS+nvA== + dependencies: + moment ">= 2.9.0" + +"moment@>= 2.6.0", "moment@>= 2.9.0", moment@^2.10, moment@^2.24.0: + version "2.29.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" + integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== + +mout@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/mout/-/mout-1.0.0.tgz#9bdf1d4af57d66d47cb353a6335a3281098e1501" + integrity sha1-m98dSvV9ZtR8s1OmM1oygQmOFQE= + +ms@0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" + integrity sha1-cIFVpeROM/X9D8U+gdDUCpG+H/8= + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +multimatch@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/multimatch/-/multimatch-4.0.0.tgz#8c3c0f6e3e8449ada0af3dd29efb491a375191b3" + integrity sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ== + dependencies: + "@types/minimatch" "^3.0.3" + array-differ "^3.0.0" + array-union "^2.1.0" + arrify "^2.0.1" + minimatch "^3.0.4" + +nanomatch@^1.2.9: + version "1.2.13" + resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119" + integrity sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA== + dependencies: + arr-diff "^4.0.0" + array-unique "^0.3.2" + define-property "^2.0.2" + extend-shallow "^3.0.2" + fragment-cache "^0.2.1" + is-windows "^1.0.2" + kind-of "^6.0.2" + object.pick "^1.3.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +native-request@^1.0.5: + version "1.0.8" + resolved "https://registry.yarnpkg.com/native-request/-/native-request-1.0.8.tgz#8f66bf606e0f7ea27c0e5995eb2f5d03e33ae6fb" + integrity sha512-vU2JojJVelUGp6jRcLwToPoWGxSx23z/0iX+I77J3Ht17rf2INGjrhOoQnjVo60nQd8wVsgzKkPfRXBiVdD2ag== + +negotiator@0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" + integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== + +neo-async@^2.6.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" + integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== + +node-releases@^1.1.70: + version "1.1.70" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.70.tgz#66e0ed0273aa65666d7fe78febe7634875426a08" + integrity sha512-Slf2s69+2/uAD79pVVQo8uSiC34+g8GWY8UH2Qtqv34ZfhYrxpYpfzs9Js9d6O0mbDmALuxaTlplnBTnSELcrw== + +nomnom@1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/nomnom/-/nomnom-1.8.1.tgz#2151f722472ba79e50a76fc125bb8c8f2e4dc2a7" + integrity sha1-IVH3Ikcrp55Qp2/BJbuMjy5Nwqc= + dependencies: + chalk "~0.4.0" + underscore "~1.6.0" + +nopt@^3.0.1, nopt@~3.0.6: + version "3.0.6" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" + integrity sha1-xkZdvwirzU2zWTF/eaxopkayj/k= + dependencies: + abbrev "1" + +nopt@~1.0.10: + version "1.0.10" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee" + integrity sha1-bd0hvSoxQXuScn3Vhfim83YI6+4= + dependencies: + abbrev "1" + +nopt@~4.0.1: + version "4.0.3" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" + integrity sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg== + dependencies: + abbrev "1" + osenv "^0.1.4" + +npmlog@4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" + integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== + dependencies: + are-we-there-yet "~1.1.2" + console-control-strings "~1.1.0" + gauge "~2.7.3" + set-blocking "~2.0.0" + +number-is-nan@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" + integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= + +oauth-sign@~0.8.1: + version "0.8.2" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" + integrity sha1-Rqarfwrq2N6unsBWV4C31O/rnUM= + +object-assign@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-2.1.1.tgz#43c36e5d569ff8e4816c4efa8be02d26967c18aa" + integrity sha1-Q8NuXVaf+OSBbE76i+AtJpZ8GKo= + +object-assign@^4.1.0, object-assign@~4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= + +object-component@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291" + integrity sha1-8MaapQ78lbhmwYb0AKM3acsvEpE= + +object-copy@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/object-copy/-/object-copy-0.1.0.tgz#7e7d858b781bd7c991a41ba975ed3812754e998c" + integrity sha1-fn2Fi3gb18mRpBupde04EnVOmYw= + dependencies: + copy-descriptor "^0.1.0" + define-property "^0.2.5" + kind-of "^3.0.3" + +object-keys@^1.0.12, object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + +object-visit@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb" + integrity sha1-95xEk68MU3e1n+OdOV5BBC3QRbs= + dependencies: + isobject "^3.0.0" + +object.assign@^4.1.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940" + integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + has-symbols "^1.0.1" + object-keys "^1.1.1" + +object.defaults@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/object.defaults/-/object.defaults-1.1.0.tgz#3a7f868334b407dea06da16d88d5cd29e435fecf" + integrity sha1-On+GgzS0B96gbaFtiNXNKeQ1/s8= + dependencies: + array-each "^1.0.1" + array-slice "^1.0.0" + for-own "^1.0.0" + isobject "^3.0.0" + +object.map@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/object.map/-/object.map-1.0.1.tgz#cf83e59dc8fcc0ad5f4250e1f78b3b81bd801d37" + integrity sha1-z4Plncj8wK1fQlDh94s7gb2AHTc= + dependencies: + for-own "^1.0.0" + make-iterator "^1.0.0" + +object.pick@^1.2.0, object.pick@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/object.pick/-/object.pick-1.3.0.tgz#87a10ac4c1694bd2e1cbf53591a66141fb5dd747" + integrity sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c= + dependencies: + isobject "^3.0.1" + +observable@~1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/observable/-/observable-1.3.1.tgz#3d2a203b1430daf6a4d6d36f236cc13e6ed5bb4e" + integrity sha1-PSogOxQw2vak1tNvI2zBPm7Vu04= + +on-finished@~2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" + integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= + dependencies: + ee-first "1.1.1" + +once@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + dependencies: + wrappy "1" + +openlayers@4.3.0-beta.3: + version "4.3.0-beta.3" + resolved "https://registry.yarnpkg.com/openlayers/-/openlayers-4.3.0-beta.3.tgz#aaaf2aa4943d8c5f7e72fd54526ecb8e96f4ba74" + integrity sha512-Lx/RSrWI+5WNBJ1i8NrTWNsMsytLDVFvLJNq54pwZ0HjwJmm8pCdDZ0CO/e7TC6NfMZspF4VvKcA1AC9P2tRzg== + dependencies: + "@mapbox/vector-tile" "1.3.0" + async "2.5.0" + closure-util "1.22.0" + fs-extra "4.0.1" + jsdoc "3.5.4" + nomnom "1.8.1" + pbf "3.0.5" + pixelworks "1.1.0" + rbush "2.0.1" + rollup "^0.45.0" + rollup-plugin-cleanup "^1.0.0" + rollup-plugin-commonjs "^8.0.2" + rollup-plugin-node-resolve "^3.0.0" + temp "0.8.3" + walk "2.3.9" + +openlayers@4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/openlayers/-/openlayers-4.3.4.tgz#0102c376b2046e28eebfd4a7cdec69322c773916" + integrity sha512-ATTYSb16sPwgjqZuGZQQOagCUy1rSCWfXNQZDkQ34YcyVObO6AMl+PukF049v0eKiSoh7zDzJX6KJWg74HI3wA== + dependencies: + "@mapbox/vector-tile" "1.3.0" + async "2.5.0" + closure-util "1.22.0" + fs-extra "4.0.1" + jsdoc "3.5.4" + nomnom "1.8.1" + pbf "3.0.5" + pixelworks "1.1.0" + rbush "2.0.1" + rollup "^0.47.2" + rollup-plugin-cleanup "^1.0.0" + rollup-plugin-commonjs "^8.0.2" + rollup-plugin-node-resolve "^3.0.0" + temp "0.8.3" + walk "2.3.9" + +optimist@*, optimist@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686" + integrity sha1-2j6nRob6IaGaERwybpDrFaAZZoY= + dependencies: + minimist "~0.0.1" + wordwrap "~0.0.2" + +os-homedir@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" + integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= + +os-tmpdir@^1.0.0, os-tmpdir@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" + integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= + +osenv@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" + integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== + dependencies: + os-homedir "^1.0.0" + os-tmpdir "^1.0.0" + +p-limit@^2.0.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" + integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== + dependencies: + p-try "^2.0.0" + +p-locate@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" + integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ== + dependencies: + p-limit "^2.0.0" + +p-try@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" + integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== + +parse-filepath@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/parse-filepath/-/parse-filepath-1.0.2.tgz#a632127f53aaf3d15876f5872f3ffac763d6c891" + integrity sha1-pjISf1Oq89FYdvWHLz/6x2PWyJE= + dependencies: + is-absolute "^1.0.0" + map-cache "^0.2.0" + path-root "^0.1.1" + +parse-passwd@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6" + integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY= + +parseqs@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/parseqs/-/parseqs-0.0.5.tgz#d5208a3738e46766e291ba2ea173684921a8b89d" + integrity sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0= + dependencies: + better-assert "~1.0.0" + +parseuri@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/parseuri/-/parseuri-0.0.5.tgz#80204a50d4dbb779bfdc6ebe2778d90e4bce320a" + integrity sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo= + dependencies: + better-assert "~1.0.0" + +pascalcase@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14" + integrity sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ= + +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU= + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + +path-parse@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" + integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + +path-root-regex@^0.1.0: + version "0.1.2" + resolved "https://registry.yarnpkg.com/path-root-regex/-/path-root-regex-0.1.2.tgz#bfccdc8df5b12dc52c8b43ec38d18d72c04ba96d" + integrity sha1-v8zcjfWxLcUsi0PsONGNcsBLqW0= + +path-root@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/path-root/-/path-root-0.1.1.tgz#9a4a6814cac1c0cd73360a95f32083c8ea4745b7" + integrity sha1-mkpoFMrBwM1zNgqV8yCDyOpHRbc= + dependencies: + path-root-regex "^0.1.0" + +patternfly-bootstrap-treeview@2.1.10: + version "2.1.10" + resolved "https://registry.yarnpkg.com/patternfly-bootstrap-treeview/-/patternfly-bootstrap-treeview-2.1.10.tgz#f96043734bb4ecac951783e2745e3f9a8873ffd4" + integrity sha512-P9+iFu34CwX+R5Fd7/EWbxTug0q9mDj53PnZIIh5ie54KX2kD0+54lCWtpD9SVylDwDtDv3n3A6gbFVkx7HsuA== + dependencies: + bootstrap "^3.4.1" + jquery "^3.4.1" + +pbf@3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/pbf/-/pbf-3.0.5.tgz#24f0fa2cbeb16e5c56a5901bb7e9c2ac0c8059be" + integrity sha1-JPD6LL6xblxWpZAbt+nCrAyAWb4= + dependencies: + ieee754 "^1.1.6" + resolve-protobuf-schema "^2.0.0" + +performance-now@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" + integrity sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU= + +pify@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231" + integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g== + +pixelworks@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/pixelworks/-/pixelworks-1.1.0.tgz#1f095ad48dca8bf8a1c8258e0092031a44f22ca5" + integrity sha1-Hwla1I3Ki/ihyCWOAJIDGkTyLKU= + +pkg-up@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5" + integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA== + dependencies: + find-up "^3.0.0" + +posix-character-classes@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" + integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs= + +pretty-bytes@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-3.0.1.tgz#27d0008d778063a0b4811bb35c79f1bd5d5fbccf" + integrity sha1-J9AAjXeAY6C0gRuzXHnxvV1fvM8= + dependencies: + number-is-nan "^1.0.0" + +process-nextick-args@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" + integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== + +proj4@^2.5.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/proj4/-/proj4-2.7.0.tgz#4250ee16699d53ebc3318befb1875e9722145f29" + integrity sha512-UVhulf8m70/dREOBrJagWq8cDYUgjQUWILRqys/gqo/+ZLeNB/04zbtPhJbz8+cCPzZNQMychfBaWUCP60U9mQ== + dependencies: + mgrs "1.0.0" + wkt-parser "^1.2.4" + +protocol-buffers-schema@^3.3.1: + version "3.5.1" + resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-3.5.1.tgz#8388e768d383ac8cbea23e1280dfadb79f4122ad" + integrity sha512-YVCvdhxWNDP8/nJDyXLuM+UFsuPk4+1PB7WGPVDzm3HTHbzFLxQYeW2iZpS4mmnXrQJGBzt230t/BbEb7PrQaw== + +prr@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" + integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= + +punycode@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= + +q@1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/q/-/q-1.5.0.tgz#dd01bac9d06d30e6f219aecb8253ee9ebdc308f1" + integrity sha1-3QG6ydBtMObyGa7LglPunr3DCPE= + +q@^1.1.2: + version "1.5.1" + resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" + integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= + +qs@^6.4.0: + version "6.9.6" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.6.tgz#26ed3c8243a431b2924aca84cc90471f35d5a0ee" + integrity sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ== + +qs@~6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" + integrity sha1-E+JtKK1rD/qpExLNO/cI7TUecjM= + +quickselect@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-1.1.1.tgz#852e412ce418f237ad5b660d70cffac647ae94c2" + integrity sha512-qN0Gqdw4c4KGPsBOQafj6yj/PA6c/L63f6CaZ/DCF/xF4Esu3jVmKLUDYxghFx8Kb/O7y9tI7x2RjTSXwdK1iQ== + +range-parser@~1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + +raty-js@^2.9.0: + version "2.9.0" + resolved "https://registry.yarnpkg.com/raty-js/-/raty-js-2.9.0.tgz#63076c3e17d6d9824d3998fe0a5d4625022db6fb" + integrity sha512-sh4w6Zebi3JGmhdc6KEPVXZVkgE75j3F/0aA/CV6uq4I06AS82y8p6Kpiwh1cnmZiWbjeARv/4XQKpio68X8RA== + dependencies: + jquery "^3.3.1" + +raw-body@~1.1.0: + version "1.1.7" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-1.1.7.tgz#1d027c2bfa116acc6623bca8f00016572a87d425" + integrity sha1-HQJ8K/oRasxmI7yo8AAWVyqH1CU= + dependencies: + bytes "1" + string_decoder "0.10" + +rbush@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/rbush/-/rbush-2.0.1.tgz#4cfaca28c3064bc0ee75431a1b79990e875eefa9" + integrity sha1-TPrKKMMGS8DudUMaG3mZDode76k= + dependencies: + quickselect "^1.0.0" + +readable-stream@1.1: + version "1.1.13" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" + integrity sha1-9u73ZPUUyJ4rniMUanW6EGdW0j4= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +readable-stream@^1.1.8: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +readable-stream@^2.0.6: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + integrity sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q= + dependencies: + resolve "^1.1.6" + +regenerate-unicode-properties@^8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec" + integrity sha512-F9DjY1vKLo/tPePDycuH3dn9H1OTPIkVD9Kz4LODu+F2C75mgjAJ7x/gwy6ZcSNRAAkhNlJSOHRe8k3p+K9WhA== + dependencies: + regenerate "^1.4.0" + +regenerate@^1.4.0: + version "1.4.2" + resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a" + integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A== + +regenerator-runtime@^0.13.4: + version "0.13.7" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz#cac2dacc8a1ea675feaabaeb8ae833898ae46f55" + integrity sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew== + +regenerator-transform@^0.14.2: + version "0.14.5" + resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.14.5.tgz#c98da154683671c9c4dcb16ece736517e1b7feb4" + integrity sha512-eOf6vka5IO151Jfsw2NO9WpGX58W6wWmefK3I1zEGr0lOD0u8rwPaNqQL1aRxUaxLeKO3ArNh3VYg1KbaD+FFw== + dependencies: + "@babel/runtime" "^7.8.4" + +regex-not@^1.0.0, regex-not@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" + integrity sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A== + dependencies: + extend-shallow "^3.0.2" + safe-regex "^1.1.0" + +regexpu-core@^4.7.1: + version "4.7.1" + resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.1.tgz#2dea5a9a07233298fbf0db91fa9abc4c6e0f8ad6" + integrity sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ== + dependencies: + regenerate "^1.4.0" + regenerate-unicode-properties "^8.2.0" + regjsgen "^0.5.1" + regjsparser "^0.6.4" + unicode-match-property-ecmascript "^1.0.4" + unicode-match-property-value-ecmascript "^1.2.0" + +regjsgen@^0.5.1: + version "0.5.2" + resolved "https://registry.yarnpkg.com/regjsgen/-/regjsgen-0.5.2.tgz#92ff295fb1deecbf6ecdab2543d207e91aa33733" + integrity sha512-OFFT3MfrH90xIW8OOSyUrk6QHD5E9JOTeGodiJeBS3J6IwlgzJMNE/1bZklWz5oTg+9dCMyEetclvCVXOPoN3A== + +regjsparser@^0.6.4: + version "0.6.7" + resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.6.7.tgz#c00164e1e6713c2e3ee641f1701c4b7aa0a7f86c" + integrity sha512-ib77G0uxsA2ovgiYbCVGx4Pv3PSttAx2vIwidqQzbL2U5S4Q+j00HdSAneSBuyVcMvEnTXMjiGgB+DlXozVhpQ== + dependencies: + jsesc "~0.5.0" + +repeat-element@^1.1.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.3.tgz#782e0d825c0c5a3bb39731f84efee6b742e6b1ce" + integrity sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g== + +repeat-string@^1.5.2, repeat-string@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" + integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= + +request-progress@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/request-progress/-/request-progress-3.0.0.tgz#4ca754081c7fec63f505e4faa825aa06cd669dbe" + integrity sha1-TKdUCBx/7GP1BeT6qCWqBs1mnb4= + dependencies: + throttleit "^1.0.0" + +request@2.81.0: + version "2.81.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" + integrity sha1-xpKJRqDgbF+Nb4qTM0af/aRimKA= + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~4.2.1" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + performance-now "^0.2.0" + qs "~6.4.0" + safe-buffer "^5.0.1" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "^0.6.0" + uuid "^3.0.0" + +requirejs-text@^2.0.15: + version "2.0.15" + resolved "https://registry.yarnpkg.com/requirejs-text/-/requirejs-text-2.0.15.tgz#13138733613fc4457b7e1247e8cb751df7aa5429" + integrity sha1-ExOHM2E/xEV7fhJH6Mt1HfeqVCk= + +requirejs@^2.3.6: + version "2.3.6" + resolved "https://registry.yarnpkg.com/requirejs/-/requirejs-2.3.6.tgz#e5093d9601c2829251258c0b9445d4d19fa9e7c9" + integrity sha512-ipEzlWQe6RK3jkzikgCupiTbTvm4S0/CAU5GlgptkN5SO6F3u0UD0K18wy6ErDqiCyP4J4YYe1HuAShvsxePLg== + +requizzle@~0.2.1: + version "0.2.3" + resolved "https://registry.yarnpkg.com/requizzle/-/requizzle-0.2.3.tgz#4675c90aacafb2c036bd39ba2daa4a1cb777fded" + integrity sha512-YanoyJjykPxGHii0fZP0uUPEXpvqfBDxWV7s6GKAiiOsiqhX6vHNyW3Qzdmqp/iq/ExbhaGbVrjB4ruEVSM4GQ== + dependencies: + lodash "^4.17.14" + +resolve-dir@^1.0.0, resolve-dir@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-1.0.1.tgz#79a40644c362be82f26effe739c9bb5382046f43" + integrity sha1-eaQGRMNivoLybv/nOcm7U4IEb0M= + dependencies: + expand-tilde "^2.0.0" + global-modules "^1.0.0" + +resolve-from@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" + integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== + +resolve-pkg@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg/-/resolve-pkg-2.0.0.tgz#ac06991418a7623edc119084edc98b0e6bf05a41" + integrity sha512-+1lzwXehGCXSeryaISr6WujZzowloigEofRB+dj75y9RRa/obVcYgbHJd53tdYw8pvZj8GojXaaENws8Ktw/hQ== + dependencies: + resolve-from "^5.0.0" + +resolve-protobuf-schema@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz#9ca9a9e69cf192bbdaf1006ec1973948aa4a3758" + integrity sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ== + dependencies: + protocol-buffers-schema "^3.3.1" + +resolve-url@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" + integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= + +resolve@^1.1.6, resolve@^1.1.7, resolve@^1.4.0: + version "1.19.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" + integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg== + dependencies: + is-core-module "^2.1.0" + path-parse "^1.0.6" + +ret@~0.1.10: + version "0.1.15" + resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" + integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg== + +retry@0.10.1: + version "0.10.1" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4" + integrity sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q= + +right-align@^0.1.1: + version "0.1.3" + resolved "https://registry.yarnpkg.com/right-align/-/right-align-0.1.3.tgz#61339b722fe6a3515689210d24e14c96148613ef" + integrity sha1-YTObci/mo1FWiSENJOFMlhSGE+8= + dependencies: + align-text "^0.1.1" + +rimraf@2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" + integrity sha1-wjOOxkPfeht/5cVPqG9XQopV8z0= + dependencies: + glob "^7.0.5" + +rimraf@^2.6.2: + version "2.7.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" + integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== + dependencies: + glob "^7.1.3" + +rimraf@~2.2.6: + version "2.2.8" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582" + integrity sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI= + +rimraf@~2.6.2: + version "2.6.3" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" + integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA== + dependencies: + glob "^7.1.3" + +rimraf@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + +rollup-plugin-cleanup@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/rollup-plugin-cleanup/-/rollup-plugin-cleanup-1.0.1.tgz#ca056c74fe6ea21783f99851963b173cbe8e9359" + integrity sha1-ygVsdP5uoheD+ZhRljsXPL6Ok1k= + dependencies: + acorn "4.x" + magic-string "*" + rollup-pluginutils "*" + +rollup-plugin-commonjs@^8.0.2: + version "8.4.1" + resolved "https://registry.yarnpkg.com/rollup-plugin-commonjs/-/rollup-plugin-commonjs-8.4.1.tgz#5c9cea2b2c3de322f5fbccd147e07ed5e502d7a0" + integrity sha512-mg+WuD+jlwoo8bJtW3Mvx7Tz6TsIdMsdhuvCnDMoyjh0oxsVgsjB/N0X984RJCWwc5IIiqNVJhXeeITcc73++A== + dependencies: + acorn "^5.2.1" + estree-walker "^0.5.0" + magic-string "^0.22.4" + resolve "^1.4.0" + rollup-pluginutils "^2.0.1" + +rollup-plugin-node-resolve@^3.0.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-3.4.0.tgz#908585eda12e393caac7498715a01e08606abc89" + integrity sha512-PJcd85dxfSBWih84ozRtBkB731OjXk0KnzN0oGp7WOWcarAFkVa71cV5hTJg2qpVsV2U8EUwrzHP3tvy9vS3qg== + dependencies: + builtin-modules "^2.0.0" + is-module "^1.0.0" + resolve "^1.1.6" + +rollup-pluginutils@*, rollup-pluginutils@^2.0.1: + version "2.8.2" + resolved "https://registry.yarnpkg.com/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz#72f2af0748b592364dbd3389e600e5a9444a351e" + integrity sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ== + dependencies: + estree-walker "^0.6.1" + +rollup@^0.45.0: + version "0.45.2" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-0.45.2.tgz#63a284c2b31234656f24e9e9717fabb6a7f0fa43" + integrity sha512-2+bq5GQSrocdhr+M92mOQRmF1evtLRzv9NdmEC2wo7BILvTG8irHCtD0q+zg8ikNu63iJicdN5IzyxAXRTFKOQ== + dependencies: + source-map-support "^0.4.0" + +rollup@^0.47.2: + version "0.47.6" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-0.47.6.tgz#83b90a1890dd3321a3f8d0b2bd216e88483f33de" + integrity sha512-bH3eWh7MzbiKTQcHQN7Ievqbs/yY7T+ZcJYboBYkp7BkRlAr2DXHPfiqlvlEH/M95giEBpinHEi/s9CVIgYT6w== + +safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.2: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +safe-json-parse@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/safe-json-parse/-/safe-json-parse-1.0.1.tgz#3e76723e38dfdda13c9b1d29a1e07ffee4b30b57" + integrity sha1-PnZyPjjf3aE8mx0poeB//uSzC1c= + +safe-regex@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-1.1.0.tgz#40a3669f3b077d1e943d44629e157dd48023bf2e" + integrity sha1-QKNmnzsHfR6UPURinhV91IAjvy4= + dependencies: + ret "~0.1.10" + +"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + +select2@4.0.10: + version "4.0.10" + resolved "https://registry.yarnpkg.com/select2/-/select2-4.0.10.tgz#c8d4e3e2504f49956b3dfd9018ab1d7bc601a41b" + integrity sha512-g9fWMdEif4eQ901eiJV3os0MG5y20/3E2WNy0PfYf0T6Q/zZB/NOroJwAxcfE6uv5x0/BGeCDGNtoj9PqCdQEg== + +select@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/select/-/select-1.1.2.tgz#0e7350acdec80b1108528786ec1d4418d11b396d" + integrity sha1-DnNQrN7ICxEIUoeG7B1EGNEbOW0= + +semver@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" + integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== + +semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: + version "5.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + +send@0.15.3: + version "0.15.3" + resolved "https://registry.yarnpkg.com/send/-/send-0.15.3.tgz#5013f9f99023df50d1bd9892c19e3defd1d53309" + integrity sha1-UBP5+ZAj31DRvZiSwZ4979HVMwk= + dependencies: + debug "2.6.7" + depd "~1.1.0" + destroy "~1.0.4" + encodeurl "~1.0.1" + escape-html "~1.0.3" + etag "~1.8.0" + fresh "0.5.0" + http-errors "~1.6.1" + mime "1.3.4" + ms "2.0.0" + on-finished "~2.3.0" + range-parser "~1.2.0" + statuses "~1.3.1" + +set-blocking@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" + integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= + +set-value@^2.0.0, set-value@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b" + integrity sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw== + dependencies: + extend-shallow "^2.0.1" + is-extendable "^0.1.1" + is-plain-object "^2.0.3" + split-string "^3.0.1" + +setprototypeof@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" + integrity sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ== + +shelljs@0.3.x: + version "0.3.0" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" + integrity sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E= + +signal-exit@^3.0.0: + version "3.0.3" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" + integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== + +snapdragon-node@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" + integrity sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw== + dependencies: + define-property "^1.0.0" + isobject "^3.0.0" + snapdragon-util "^3.0.1" + +snapdragon-util@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/snapdragon-util/-/snapdragon-util-3.0.1.tgz#f956479486f2acd79700693f6f7b805e45ab56e2" + integrity sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ== + dependencies: + kind-of "^3.2.0" + +snapdragon@^0.8.1: + version "0.8.2" + resolved "https://registry.yarnpkg.com/snapdragon/-/snapdragon-0.8.2.tgz#64922e7c565b0e14204ba1aa7d6964278d25182d" + integrity sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg== + dependencies: + base "^0.11.1" + debug "^2.2.0" + define-property "^0.2.5" + extend-shallow "^2.0.1" + map-cache "^0.2.2" + source-map "^0.5.6" + source-map-resolve "^0.5.0" + use "^3.1.0" + +sntp@1.x.x: + version "1.0.9" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-1.0.9.tgz#6541184cc90aeea6c6e7b35e2659082443c66198" + integrity sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg= + dependencies: + hoek "2.x.x" + +socket.io-adapter@~1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-1.1.2.tgz#ab3f0d6f66b8fc7fca3959ab5991f82221789be9" + integrity sha512-WzZRUj1kUjrTIrUKpZLEzFZ1OLj5FwLlAFQs9kuZJzJi5DKdU7FsWc36SNmA8iDOtwBQyT8FkrriRM8vXLYz8g== + +socket.io-client@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-2.0.1.tgz#1d52f8c74660c68bb6695953fa119971155fad93" + integrity sha1-HVL4x0Zgxou2aVlT+hGZcRVfrZM= + dependencies: + backo2 "1.0.2" + base64-arraybuffer "0.1.5" + component-bind "1.0.0" + component-emitter "1.2.1" + debug "2.6.4" + engine.io-client "~3.1.0" + has-cors "1.1.0" + indexof "0.0.1" + object-component "0.0.3" + parseuri "0.0.5" + socket.io-parser "~3.1.1" + to-array "0.1.4" + +socket.io-parser@~3.1.1: + version "3.1.3" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-3.1.3.tgz#ed2da5ee79f10955036e3da413bfd7f1e4d86c8e" + integrity sha512-g0a2HPqLguqAczs3dMECuA1RgoGFPyvDqcbaDEdCWY9g59kdUAz3YRmaJBNKXflrHNwB7Q12Gkf/0CZXfdHR7g== + dependencies: + component-emitter "1.2.1" + debug "~3.1.0" + has-binary2 "~1.0.2" + isarray "2.0.1" + +socket.io@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-2.0.1.tgz#064c125178462e477a6df23d2fdad18dd1c5914f" + integrity sha1-BkwSUXhGLkd6bfI9L9rRjdHFkU8= + dependencies: + debug "~2.6.6" + engine.io "~3.1.0" + object-assign "~4.1.1" + socket.io-adapter "~1.1.0" + socket.io-client "2.0.1" + socket.io-parser "~3.1.1" + +sortable@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/sortable/-/sortable-2.0.0.tgz#34b3542b42904c59b8454f55148a0f4f60186943" + integrity sha1-NLNUK0KQTFm4RU9VFIoPT2AYaUM= + dependencies: + hyperscript "~1.0.1" + jquery-browserify "~1.8.1" + observable "~1.3.1" + +source-map-resolve@^0.5.0: + version "0.5.3" + resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a" + integrity sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw== + dependencies: + atob "^2.1.2" + decode-uri-component "^0.2.0" + resolve-url "^0.2.1" + source-map-url "^0.4.0" + urix "^0.1.0" + +source-map-support@^0.4.0: + version "0.4.18" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.4.18.tgz#0286a6de8be42641338594e97ccea75f0a2c585f" + integrity sha512-try0/JqxPLF9nOjvSta7tVondkP5dwgyLDjVoyMDlmjugT2lRZ1OfsrYTkCd2hkDnJTKRbO/Rl3orm8vlsUzbA== + dependencies: + source-map "^0.5.6" + +source-map-url@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.1.tgz#0af66605a745a5a2f91cf1bbf8a7afbc283dec56" + integrity sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw== + +source-map@^0.4.4: + version "0.4.4" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" + integrity sha1-66T12pwNyZneaAMti092FzZSA2s= + dependencies: + amdefine ">=0.0.4" + +source-map@^0.5.0, source-map@^0.5.3, source-map@^0.5.6, source-map@~0.5.1: + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" + integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= + +source-map@^0.6.1, source-map@~0.6.0: + version "0.6.1" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + +sourcemap-codec@^1.4.4: + version "1.4.8" + resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4" + integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== + +split-string@^3.0.1, split-string@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" + integrity sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw== + dependencies: + extend-shallow "^3.0.0" + +sprintf-js@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673" + integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug== + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= + +sshpk@^1.7.0: + version "1.16.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" + integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + +static-extend@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" + integrity sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY= + dependencies: + define-property "^0.2.5" + object-copy "^0.1.0" + +"statuses@>= 1.4.0 < 2": + version "1.5.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" + integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= + +statuses@~1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" + integrity sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4= + +string-natural-compare@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-2.0.3.tgz#9dbe1dd65490a5fe14f7a5c9bc686fc67cb9c6e4" + integrity sha512-4Kcl12rNjc+6EKhY8QyDVuQTAlMWwRiNbsxnVwBUKFr7dYPQuXVrtNU4sEkjF9LHY0AY6uVbB3ktbkIH4LC+BQ== + +string-template@~0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/string-template/-/string-template-0.2.1.tgz#42932e598a352d01fc22ec3367d9d84eec6c9add" + integrity sha1-QpMuWYo1LQH8IuwzZ9nYTuxsmt0= + +string-width@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" + integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M= + dependencies: + code-point-at "^1.0.0" + is-fullwidth-code-point "^1.0.0" + strip-ansi "^3.0.0" + +"string-width@^1.0.2 || 2": + version "2.1.1" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" + integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== + dependencies: + is-fullwidth-code-point "^2.0.0" + strip-ansi "^4.0.0" + +string_decoder@0.10, string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +stringstream@~0.0.4: + version "0.0.6" + resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.6.tgz#7880225b0d4ad10e30927d167a1d6f2fd3b33a72" + integrity sha512-87GEBAkegbBcweToUrdzf3eLhWNg06FJTebl4BVJz/JgWy8CvEr9dRtX5qWphiynMSQlxxi+QqN0z5T32SLlhA== + +strip-ansi@^3.0.0, strip-ansi@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" + integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8= + dependencies: + ansi-regex "^2.0.0" + +strip-ansi@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" + integrity sha1-qEeQIusaw2iocTibY1JixQXuNo8= + dependencies: + ansi-regex "^3.0.0" + +strip-ansi@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-0.1.1.tgz#39e8a98d044d150660abe4a6808acf70bb7bc991" + integrity sha1-OeipjQRNFQZgq+SmgIrPcLt7yZE= + +strip-json-comments@1.0.x: + version "1.0.4" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-1.0.4.tgz#1e15fbcac97d3ee99bf2d73b4c656b082bbafb91" + integrity sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E= + +strip-json-comments@~2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" + integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= + +supports-color@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" + integrity sha1-U10EXOa2Nj+kARcIRimZXp3zJMc= + +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== + dependencies: + has-flag "^3.0.0" + +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +taffydb@2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/taffydb/-/taffydb-2.6.2.tgz#7cbcb64b5a141b6a2efc2c5d2c67b4e150b2a268" + integrity sha1-fLy2S1oUG2ou/CxdLGe04VCyomg= + +tar@3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/tar/-/tar-3.1.5.tgz#4981e97ab7bad4cb1d5da9232047c9047a681aef" + integrity sha512-TKJKz1fqBOZBaIQ/MGRKU0EnTGmKMLy4ReTRgP10AgtfOWBbj9PBg4MgY80GFpqGbs2EzcIctW5gbwbP4woDYg== + dependencies: + minipass "^2.0.2" + minizlib "^1.0.3" + mkdirp "^0.5.0" + yallist "^3.0.2" + +temp@*: + version "0.9.4" + resolved "https://registry.yarnpkg.com/temp/-/temp-0.9.4.tgz#cd20a8580cb63635d0e4e9d4bd989d44286e7620" + integrity sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA== + dependencies: + mkdirp "^0.5.1" + rimraf "~2.6.2" + +temp@0.8.3: + version "0.8.3" + resolved "https://registry.yarnpkg.com/temp/-/temp-0.8.3.tgz#e0c6bc4d26b903124410e4fed81103014dfc1f59" + integrity sha1-4Ma8TSa5AxJEEOT+2BEDAU38H1k= + dependencies: + os-tmpdir "^1.0.0" + rimraf "~2.2.6" + +throttleit@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" + integrity sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw= + +timeago@^1.6.7: + version "1.6.7" + resolved "https://registry.yarnpkg.com/timeago/-/timeago-1.6.7.tgz#afd467c29a911e697fc22a81888c7c3022783cb5" + integrity sha512-FikcjN98+ij0siKH4VO4dZ358PR3oDDq4Vdl1+sN9gWz1/+JXGr3uZbUShYH/hL7bMhcTpPbplJU5Tej4b4jbQ== + dependencies: + jquery ">=1.5.0 <4.0" + +tiny-binary-search@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/tiny-binary-search/-/tiny-binary-search-1.0.3.tgz#9d52e3d16dd1171eb74486caf704ba08c0c62186" + integrity sha512-STSHX/L5nI9WTLv6wrzJbAPbO7OIISX83KFBh2GVbX1Uz/vgZOU/ANn/8iV6t35yMTpoPzzO+3OQid3mifE0CA== + +tiny-emitter@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/tiny-emitter/-/tiny-emitter-2.1.0.tgz#1d1a56edfc51c43e863cbb5382a72330e3555423" + integrity sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q== + +tiny-lr@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/tiny-lr/-/tiny-lr-1.1.1.tgz#9fa547412f238fedb068ee295af8b682c98b2aab" + integrity sha512-44yhA3tsaRoMOjQQ+5v5mVdqef+kH6Qze9jTpqtVufgYjYt08zyZAwNwwVBj3i1rJMnR52IxOW0LK0vBzgAkuA== + dependencies: + body "^5.1.0" + debug "^3.1.0" + faye-websocket "~0.10.0" + livereload-js "^2.3.0" + object-assign "^4.1.0" + qs "^6.4.0" + +tinymce@^5.5.0: + version "5.6.2" + resolved "https://registry.yarnpkg.com/tinymce/-/tinymce-5.6.2.tgz#4c20c22e328c5d0baab204928dbc2b0ecbed18b4" + integrity sha512-z7zvM5seOPiW86/vqf08kStwW5Zs5U9oQfuqh2rTj4jEcT2QzxT0v72i2zw3W6rbTLldkAej6edFZphj5ee5zg== + +tmp@0.0.31: + version "0.0.31" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.31.tgz#8f38ab9438e17315e5dbd8b3657e8bfb277ae4a7" + integrity sha1-jzirlDjhcxXl29izZX6L+yd65Kc= + dependencies: + os-tmpdir "~1.0.1" + +to-array@0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/to-array/-/to-array-0.1.4.tgz#17e6c11f73dd4f3d74cda7a4ff3238e9ad9bf890" + integrity sha1-F+bBH3PdTz10zaek/zI46a2b+JA= + +to-fast-properties@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= + +to-object-path@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af" + integrity sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68= + dependencies: + kind-of "^3.0.2" + +to-regex-range@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38" + integrity sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg= + dependencies: + is-number "^3.0.0" + repeat-string "^1.6.1" + +to-regex@^3.0.1, to-regex@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/to-regex/-/to-regex-3.0.2.tgz#13cfdd9b336552f30b51f33a8ae1b42a7a7599ce" + integrity sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw== + dependencies: + define-property "^2.0.2" + extend-shallow "^3.0.2" + regex-not "^1.0.2" + safe-regex "^1.1.0" + +touch@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/touch/-/touch-0.0.3.tgz#51aef3d449571d4f287a5d87c9c8b49181a0db1d" + integrity sha1-Ua7z1ElXHU8oel2Hyci0kYGg2x0= + dependencies: + nopt "~1.0.10" + +tough-cookie@~2.3.0: + version "2.3.4" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.4.tgz#ec60cee38ac675063ffc97a5c18970578ee83655" + integrity sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA== + dependencies: + punycode "^1.4.1" + +"traverse@>=0.3.0 <0.4": + version "0.3.9" + resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" + integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk= + +tree-filter@^0.0.4: + version "0.0.4" + resolved "https://registry.yarnpkg.com/tree-filter/-/tree-filter-0.0.4.tgz#be151244575f8255ece5c18d7519cc53e25aecab" + integrity sha1-vhUSRFdfglXs5cGNdRnMU+Ja7Ks= + dependencies: + array-union "^1.0.1" + intersect "^0.1.0" + object-assign "^2.0.0" + +tree-multiselect@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/tree-multiselect/-/tree-multiselect-2.6.1.tgz#9a16a2bb24927499b10a4eea825d5a107a542240" + integrity sha512-uaj7A1AehIy5Fouve8lUlLjikPnXcOkcrx0PLRXcSn/h/wI54mfvPPxcI9xwtbZFrGK+ldNp5cEuZ9RR2xB7+g== + +tslib@^1.10.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" + integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= + +uglify-js@^2.6: + version "2.8.29" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" + integrity sha1-KcVzMUgFe7Th913zW3qcty5qWd0= + dependencies: + source-map "~0.5.1" + yargs "~3.10.0" + optionalDependencies: + uglify-to-browserify "~1.0.0" + +uglify-js@^3.1.4, uglify-js@^3.5.0: + version "3.12.6" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.12.6.tgz#f884584fcc42e10bca70db5cb32e8625c2c42535" + integrity sha512-aqWHe3DfQmZUDGWBbabZ2eQnJlQd1fKlMUu7gV+MiTuDzdgDw31bI3wA2jLLsV/hNcDP26IfyEgSVoft5+0SVw== + +uglify-to-browserify@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz#6e0924d6bda6b5afe349e39a6d632850a0f882b7" + integrity sha1-bgkk1r2mta/jSeOabWMoUKD4grc= + +ultron@~1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/ultron/-/ultron-1.1.1.tgz#9fe1536a10a664a65266a1e3ccf85fd36302bc9c" + integrity sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og== + +unc-path-regex@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa" + integrity sha1-5z3T17DXxe2G+6xrCufYxqadUPo= + +underscore.string@~3.3.5: + version "3.3.5" + resolved "https://registry.yarnpkg.com/underscore.string/-/underscore.string-3.3.5.tgz#fc2ad255b8bd309e239cbc5816fd23a9b7ea4023" + integrity sha512-g+dpmgn+XBneLmXXo+sGlW5xQEt4ErkS3mgeN2GFbremYeMBSJKr9Wf2KJplQVaiPY/f7FN6atosWYNm9ovrYg== + dependencies: + sprintf-js "^1.0.3" + util-deprecate "^1.0.2" + +underscore@^1.9.1: + version "1.12.0" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.0.tgz#4814940551fc80587cef7840d1ebb0f16453be97" + integrity sha512-21rQzss/XPMjolTiIezSu3JAjgagXKROtNrYFEOWK109qY1Uv2tVjPTZ1ci2HgvQDA16gHYSthQIJfB+XId/rQ== + +underscore@~1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" + integrity sha1-izixDKze9jM3uLJOT/htRa6lKag= + +underscore@~1.8.3: + version "1.8.3" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" + integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI= + +unicode-canonical-property-names-ecmascript@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" + integrity sha512-jDrNnXWHd4oHiTZnx/ZG7gtUTVp+gCcTTKr8L0HjlwphROEW3+Him+IpvC+xcJEFegapiMZyZe02CyuOnRmbnQ== + +unicode-match-property-ecmascript@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-1.0.4.tgz#8ed2a32569961bce9227d09cd3ffbb8fed5f020c" + integrity sha512-L4Qoh15vTfntsn4P1zqnHulG0LdXgjSO035fEpdtp6YxXhMT51Q6vgM5lYdG/5X3MjS+k/Y9Xw4SFCY9IkR0rg== + dependencies: + unicode-canonical-property-names-ecmascript "^1.0.4" + unicode-property-aliases-ecmascript "^1.0.4" + +unicode-match-property-value-ecmascript@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.2.0.tgz#0d91f600eeeb3096aa962b1d6fc88876e64ea531" + integrity sha512-wjuQHGQVofmSJv1uVISKLE5zO2rNGzM/KCYZch/QQvez7C1hUhBIuZ701fYXExuufJFMPhv2SyL8CyoIfMLbIQ== + +unicode-property-aliases-ecmascript@^1.0.4: + version "1.1.0" + resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz#dd57a99f6207bedff4628abefb94c50db941c8f4" + integrity sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg== + +union-value@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847" + integrity sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg== + dependencies: + arr-union "^3.1.0" + get-value "^2.0.6" + is-extendable "^0.1.1" + set-value "^2.0.1" + +universalify@^0.1.0: + version "0.1.2" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" + integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== + +unset-value@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559" + integrity sha1-g3aHP30jNRef+x5vw6jtDfyKtVk= + dependencies: + has-value "^0.3.1" + isobject "^3.0.0" + +uri-path@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/uri-path/-/uri-path-1.0.0.tgz#9747f018358933c31de0fccfd82d138e67262e32" + integrity sha1-l0fwGDWJM8Md4PzP2C0TjmcmLjI= + +urix@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" + integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= + +use@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" + integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== + +util-deprecate@^1.0.2, util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + +uuid@^3.0.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + +uws@~9.14.0: + version "9.14.0" + resolved "https://registry.yarnpkg.com/uws/-/uws-9.14.0.tgz#fac8386befc33a7a3705cbd58dc47b430ca4dd95" + integrity sha512-HNMztPP5A1sKuVFmdZ6BPVpBQd5bUjNC8EFMFiICK+oho/OQsAJy5hnIx4btMHiOk8j04f/DbIlqnEZ9d72dqg== + +v8flags@~3.1.1: + version "3.1.3" + resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-3.1.3.tgz#fc9dc23521ca20c5433f81cc4eb9b3033bb105d8" + integrity sha512-amh9CCg3ZxkzQ48Mhcb8iX7xpAfYJgePHxWMQCBWECpOSqJUXgY26ncA61UTV0BkPqfhcy6mzwCIoP4ygxpW8w== + dependencies: + homedir-polyfill "^1.0.1" + +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + +vlq@^0.2.2: + version "0.2.3" + resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26" + integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow== + +walk@2.3.9: + version "2.3.9" + resolved "https://registry.yarnpkg.com/walk/-/walk-2.3.9.tgz#31b4db6678f2ae01c39ea9fb8725a9031e558a7b" + integrity sha1-MbTbZnjyrgHDnqn7hyWpAx5Vins= + dependencies: + foreachasync "^3.0.0" + +websocket-driver@>=0.5.1: + version "0.7.4" + resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.7.4.tgz#89ad5295bbf64b480abcba31e4953aca706f5760" + integrity sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg== + dependencies: + http-parser-js ">=0.5.1" + safe-buffer ">=5.1.0" + websocket-extensions ">=0.1.1" + +websocket-extensions@>=0.1.1: + version "0.1.4" + resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" + integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== + +which@^1.2.14, which@~1.3.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" + integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== + dependencies: + isexe "^2.0.0" + +wide-align@^1.1.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457" + integrity sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA== + dependencies: + string-width "^1.0.2 || 2" + +window-size@0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d" + integrity sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0= + +wkt-parser@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/wkt-parser/-/wkt-parser-1.2.4.tgz#e42ba7b96f13aef82a2da9056e2d87da0fec5393" + integrity sha512-ZzKnc7ml/91fOPh5bANBL4vUlWPIYYv11waCtWTkl2TRN+LEmBg60Q1MA8gqV4hEp4MGfSj9JiHz91zw/gTDXg== + +wordwrap@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.2.tgz#b79669bb42ecb409f83d583cad52ca17eaa1643f" + integrity sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8= + +wordwrap@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" + integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= + +wordwrap@~0.0.2: + version "0.0.3" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107" + integrity sha1-o9XabNXAvAAI03I0u68b7WMFkQc= + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + +ws@~3.3.1: + version "3.3.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-3.3.3.tgz#f1cf84fe2d5e901ebce94efaece785f187a228f2" + integrity sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA== + dependencies: + async-limiter "~1.0.0" + safe-buffer "~5.1.0" + ultron "~1.1.0" + +xmlcreate@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/xmlcreate/-/xmlcreate-1.0.2.tgz#fa6bf762a60a413fb3dd8f4b03c5b269238d308f" + integrity sha1-+mv3YqYKQT+z3Y9LA8WyaSONMI8= + +xmlhttprequest-ssl@~1.5.4: + version "1.5.5" + resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz#c2876b06168aadc40e57d97e81191ac8f4398b3e" + integrity sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4= + +xtend@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" + integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== + +yallist@^3.0.0, yallist@^3.0.2: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" + integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== + +yargs@~3.10.0: + version "3.10.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-3.10.0.tgz#f7ee7bd857dd7c1d2d38c0e74efbd681d1431fd1" + integrity sha1-9+572FfdfB0tOMDnTvvWgdFDH9E= + dependencies: + camelcase "^1.0.2" + cliui "^2.1.0" + decamelize "^1.0.0" + window-size "0.1.0" + +yeast@0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/yeast/-/yeast-0.1.2.tgz#008e06d8094320c372dbc2f8ed76a0ca6c8ac419" + integrity sha1-AI4G2AlDIMNy28L47XagymyKxBk= + +zeroclipboard@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/zeroclipboard/-/zeroclipboard-2.3.0.tgz#592ebd833a4308688b0739697d3dbf989002c9af" + integrity sha1-WS69gzpDCGiLBzlpfT2/mJACya8= diff --git a/geonode/client/templates/ol/datasets/dataset_ol2_map.html b/geonode/client/templates/ol/datasets/dataset_ol2_map.html new file mode 100644 index 0000000..5912130 --- /dev/null +++ b/geonode/client/templates/ol/datasets/dataset_ol2_map.html @@ -0,0 +1,83 @@ + +{% load static %} + + + + + + + diff --git a/geonode/client/templates/ol/maps/map_ol2.html b/geonode/client/templates/ol/maps/map_ol2.html new file mode 100644 index 0000000..e38ff19 --- /dev/null +++ b/geonode/client/templates/ol/maps/map_ol2.html @@ -0,0 +1,59 @@ +{% load static %} + + + + + + + diff --git a/geonode/client/templatetags/__init__.py b/geonode/client/templatetags/__init__.py new file mode 100644 index 0000000..68c7b7d --- /dev/null +++ b/geonode/client/templatetags/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/client/templatetags/client_lib_tags.py b/geonode/client/templatetags/client_lib_tags.py new file mode 100644 index 0000000..2ec9837 --- /dev/null +++ b/geonode/client/templatetags/client_lib_tags.py @@ -0,0 +1,234 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django import template +from django.conf import settings +from django.template.base import FilterExpression, kwarg_re +from ..hooks import hookset + +register = template.Library() + + +@register.simple_tag +def mapbox_access_token(): + return getattr(settings, "MAPBOX_ACCESS_TOKEN", None) + + +@register.simple_tag +def bing_api_key(): + return getattr(settings, "BING_API_KEY", None) + + +@register.simple_tag +def google_api_key(): + return getattr(settings, "GOOGLE_API_KEY", None) + + +# For client single page links +@register.simple_tag +def dataset_list_url(): + return hookset.dataset_list_url() + + +@register.simple_tag +def dataset_upload_url(): + return hookset.dataset_upload_url() + + +@register.simple_tag +def dataset_detail_url(layer): + return hookset.dataset_detail_url(layer) + + +@register.simple_tag +def map_list_url(): + return hookset.map_list_url() + + +@register.simple_tag +def map_detail_url(map): + return hookset.map_detail_url(map) + + +@register.simple_tag +def document_list_url(): + return hookset.document_list_url() + + +@register.simple_tag +def document_detail_url(document): + return hookset.document_detail_url(document) + + +@register.simple_tag +def geoapp_list_url(): + return hookset.geoapp_list_url() + + +@register.simple_tag +def geoapp_detail_url(geoapp): + return hookset.geoapp_detail_url(geoapp) + + +def parse_tag(token, parser): + """ + Generic template tag parser. + + Returns a three-tuple: (tag_name, args, kwargs) + + tag_name is a string, the name of the tag. + + args is a list of FilterExpressions, from all the arguments that didn't look like kwargs, + in the order they occurred, including any that were mingled amongst kwargs. + + kwargs is a dictionary mapping kwarg names to FilterExpressions, for all the arguments that + looked like kwargs, including any that were mingled amongst args. + + (At rendering time, a FilterExpression f can be evaluated by calling f.resolve(context).) + """ + # Split the tag content into words, respecting quoted strings. + bits = token.split_contents() + + # Pull out the tag name. + tag_name = bits.pop(0) + + # Parse the rest of the args, and build FilterExpressions from them so that + # we can evaluate them later. + args = [] + kwargs = {} + for bit in bits: + # Is this a kwarg or an arg? + match = kwarg_re.match(bit) + kwarg_format = match and match.group(1) + if kwarg_format: + key, value = match.groups() + kwargs[key] = FilterExpression(value, parser) + else: + args.append(FilterExpression(bit, parser)) + + return (tag_name, args, kwargs) + + +class GeoNodeClientLibraryTag(template.Node): + def __init__(self, tag_name, *args, **kwargs): + self.tag_name = tag_name + + def render(self, context): + t = None + + # LAYERS + if self.tag_name == "get_dataset_list": + t = context.template.engine.get_template(hookset.dataset_list_template(context=context)) + elif self.tag_name == "get_dataset_detail": + t = context.template.engine.get_template("geonode-mapstore-client/legacy/dataset_detail.html") + elif self.tag_name == "get_dataset_new": + t = context.template.engine.get_template(hookset.dataset_new_template(context=context)) + elif self.tag_name == "get_dataset_view": + t = context.template.engine.get_template(hookset.dataset_view_template(context=context)) + elif self.tag_name == "get_dataset_edit": + t = context.template.engine.get_template(hookset.dataset_edit_template(context=context)) + elif self.tag_name == "get_dataset_update": + t = context.template.engine.get_template(hookset.dataset_update_template(context=context)) + elif self.tag_name == "get_dataset_embed": + t = context.template.engine.get_template(hookset.dataset_embed_template(context=context)) + elif self.tag_name == "get_dataset_download": + t = context.template.engine.get_template(hookset.dataset_download_template(context=context)) + elif self.tag_name == "get_dataset_style_edit": + t = context.template.engine.get_template(hookset.dataset_style_edit_template(context=context)) + + # MAPS + if self.tag_name == "get_map_list": + t = context.template.engine.get_template(hookset.map_list_template(context=context)) + elif self.tag_name == "get_map_detail": + t = context.template.engine.get_template(hookset.map_detail_template(context=context)) + elif self.tag_name == "get_map_new": + t = context.template.engine.get_template(hookset.map_new_template(context=context)) + elif self.tag_name == "get_map_view": + t = context.template.engine.get_template(hookset.map_view_template(context=context)) + elif self.tag_name == "get_map_edit": + t = context.template.engine.get_template(hookset.map_edit_template(context=context)) + elif self.tag_name == "get_map_update": + t = context.template.engine.get_template(hookset.map_update_template(context=context)) + elif self.tag_name == "get_map_embed": + t = context.template.engine.get_template("geonode-mapstore-client/map_embed.html") + elif self.tag_name == "get_map_download": + t = context.template.engine.get_template(hookset.map_download_template(context=context)) + + # GEONODE_APPS + if self.tag_name == "get_geoapp_list": + t = context.template.engine.get_template(hookset.geoapp_list_template(context=context)) + elif self.tag_name == "get_geoapp_detail": + t = context.template.engine.get_template(hookset.geoapp_detail_template(context=context)) + elif self.tag_name == "get_geoapp_new": + t = context.template.engine.get_template(hookset.geoapp_new_template(context=context)) + elif self.tag_name == "get_geoapp_view": + t = context.template.engine.get_template(hookset.geoapp_view_template(context=context)) + elif self.tag_name == "get_geoapp_edit": + t = context.template.engine.get_template(hookset.geoapp_edit_template(context=context)) + elif self.tag_name == "get_geoapp_update": + t = context.template.engine.get_template(hookset.geoapp_update_template(context=context)) + elif self.tag_name == "get_geoapp_embed": + t = context.template.engine.get_template(hookset.geoapp_embed_template(context=context)) + elif self.tag_name == "get_geoapp_download": + t = context.template.engine.get_template(hookset.geoapp_download_template(context=context)) + + # 3DTILES + + if self.tag_name == "get_resourcebase_embed": + t = context.template.engine.get_template(hookset.resourcebase_embed_template(context=context)) + + if t: + return t.render(context) + else: + return "" + + +def do_get_client_library_template(parser, token): + tag_name, args, kwargs = parse_tag(token, parser) + return GeoNodeClientLibraryTag(tag_name, args, kwargs) + + +register.tag("get_dataset_list", do_get_client_library_template) +register.tag("get_dataset_detail", do_get_client_library_template) +register.tag("get_dataset_new", do_get_client_library_template) +register.tag("get_dataset_view", do_get_client_library_template) +register.tag("get_dataset_edit", do_get_client_library_template) +register.tag("get_dataset_update", do_get_client_library_template) +register.tag("get_dataset_embed", do_get_client_library_template) +register.tag("get_dataset_download", do_get_client_library_template) +register.tag("get_dataset_style_edit", do_get_client_library_template) + +register.tag("get_map_list", do_get_client_library_template) +register.tag("get_map_detail", do_get_client_library_template) +register.tag("get_map_new", do_get_client_library_template) +register.tag("get_map_view", do_get_client_library_template) +register.tag("get_map_edit", do_get_client_library_template) +register.tag("get_map_update", do_get_client_library_template) +register.tag("get_map_embed", do_get_client_library_template) +register.tag("get_map_download", do_get_client_library_template) + +register.tag("get_geoapp_list", do_get_client_library_template) +register.tag("get_geoapp_detail", do_get_client_library_template) +register.tag("get_geoapp_new", do_get_client_library_template) +register.tag("get_geoapp_view", do_get_client_library_template) +register.tag("get_geoapp_edit", do_get_client_library_template) +register.tag("get_geoapp_update", do_get_client_library_template) +register.tag("get_geoapp_embed", do_get_client_library_template) +register.tag("get_geoapp_download", do_get_client_library_template) + +register.tag("get_resourcebase_embed", do_get_client_library_template) diff --git a/geonode/client/tests.py b/geonode/client/tests.py new file mode 100644 index 0000000..8e156a6 --- /dev/null +++ b/geonode/client/tests.py @@ -0,0 +1,20 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +# this file is a placeholder as paver tasks expect a tests.py module for each app diff --git a/geonode/compat.py b/geonode/compat.py new file mode 100644 index 0000000..411c2cc --- /dev/null +++ b/geonode/compat.py @@ -0,0 +1,41 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + + +long, unicode, basestring = int, str, str +unicode = str +string_type = str + + +def ensure_string(payload_bytes): + import re + + _payload = payload_bytes + try: + _payload = payload_bytes.decode("utf-8") + except AttributeError: + # when _payload is already a string + pass + except UnicodeDecodeError: + # when payload is a byte-like object (e.g bytearray) + # primarily used in when _payload is an image + return _payload + if re.match(r"b\'(.*)\'", _payload): + _payload = re.match(r"b\'(.*)\'", _payload).groups()[0] + return _payload diff --git a/geonode/context_processors.py b/geonode/context_processors.py new file mode 100644 index 0000000..8b83ba2 --- /dev/null +++ b/geonode/context_processors.py @@ -0,0 +1,119 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import warnings + +from django.conf import settings +from django.contrib.sites.models import Site + +from geonode import get_version +from geonode.catalogue import default_catalogue_backend +from geonode.notifications_helper import has_notifications +from geonode.base.models import Configuration, Thesaurus +from geonode.utils import get_geonode_app_types + +from allauth.socialaccount.models import SocialApp + + +def resource_urls(request): + """Global values to pass to templates""" + site = Site.objects.get_current() + thesaurus = Thesaurus.objects.filter(facet=True).all().order_by("order", "id") + if hasattr(settings, "THESAURUS"): + warnings.warn( + "Thesaurus settings is going to be" + "deprecated in the future versions, please move the settings to " + "the new configuration ", + FutureWarning, + ) + defaults = dict( + STATIC_URL=settings.STATIC_URL, + CATALOGUE_BASE_URL=default_catalogue_backend()["URL"], + ACCOUNT_OPEN_SIGNUP=settings.ACCOUNT_OPEN_SIGNUP, + ACCOUNT_APPROVAL_REQUIRED=settings.ACCOUNT_APPROVAL_REQUIRED, + VERSION=get_version(), + SITE_NAME=site.name, + SITE_DOMAIN=site.domain, + SITEURL=settings.SITEURL, + INSTALLED_APPS=settings.INSTALLED_APPS, + THEME_ACCOUNT_CONTACT_EMAIL=settings.THEME_ACCOUNT_CONTACT_EMAIL, + TINYMCE_DEFAULT_CONFIG=settings.TINYMCE_DEFAULT_CONFIG, + DEBUG_STATIC=getattr(settings, "DEBUG_STATIC", False), + PROXY_URL=getattr(settings, "PROXY_URL", "/proxy/?url="), + DISPLAY_SOCIAL=getattr(settings, "DISPLAY_SOCIAL", False), + DISPLAY_RATINGS=getattr(settings, "DISPLAY_RATINGS", False), + DISPLAY_WMS_LINKS=getattr(settings, "DISPLAY_WMS_LINKS", True), + CREATE_LAYER=getattr(settings, "CREATE_LAYER", True), + TWITTER_CARD=getattr(settings, "TWITTER_CARD", False), + TWITTER_SITE=getattr(settings, "TWITTER_SITE", "@GeoNode"), + TWITTER_HASHTAGS=getattr(settings, "TWITTER_HASHTAGS", []), + OPENGRAPH_ENABLED=getattr(settings, "OPENGRAPH_ENABLED", False), + ADMIN_MODERATE_UPLOADS=getattr(settings, "ADMIN_MODERATE_UPLOADS", False), + TOPICCATEGORY_MANDATORY=getattr(settings, "TOPICCATEGORY_MANDATORY", False), + GROUP_MANDATORY_RESOURCES=getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + GROUP_PRIVATE_RESOURCES=getattr(settings, "GROUP_PRIVATE_RESOURCES", False), + RESOURCE_PUBLISHING=getattr(settings, "RESOURCE_PUBLISHING", False), + SKIP_PERMS_FILTER=getattr(settings, "SKIP_PERMS_FILTER", False), + CLIENT_RESULTS_LIMIT=getattr(settings, "CLIENT_RESULTS_LIMIT", 10), + API_LIMIT_PER_PAGE=getattr(settings, "API_LIMIT_PER_PAGE", 20), + SRID_DETAIL=getattr(settings, "SRID", dict()).get("DETAIL", "never"), + LICENSES_ENABLED=getattr(settings, "LICENSES", dict()).get("ENABLED", False), + LICENSES_DETAIL=getattr(settings, "LICENSES", dict()).get("DETAIL", "never"), + LICENSES_METADATA=getattr(settings, "LICENSES", dict()).get("METADATA", "never"), + USE_GEOSERVER=getattr(settings, "USE_GEOSERVER", False), + USE_NOTIFICATIONS=has_notifications, + USE_MONITORING=settings.MONITORING_ENABLED, + DEFAULT_ANONYMOUS_VIEW_PERMISSION=getattr(settings, "DEFAULT_ANONYMOUS_VIEW_PERMISSION", False), + DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=getattr(settings, "DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION", False), + EXIF_ENABLED=getattr(settings, "EXIF_ENABLED", False), + FAVORITE_ENABLED=getattr(settings, "FAVORITE_ENABLED", False), + SEARCH_FILTERS=getattr(settings, "SEARCH_FILTERS", False), + THESAURI_FILTERS=( + [ + t["name"] + for t in [ + settings.THESAURUS, + ] + if t.get("filter") + ] + if hasattr(settings, "THESAURUS") + else [t.identifier for t in thesaurus] + ), + MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS=getattr(settings, "MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS", False), + SHOW_PROFILE_EMAIL=getattr(settings, "SHOW_PROFILE_EMAIL", False), + OGC_SERVER=getattr(settings, "OGC_SERVER", None), + DELAYED_SECURITY_SIGNALS=getattr(settings, "DELAYED_SECURITY_SIGNALS", False), + READ_ONLY_MODE=getattr(Configuration.load(), "read_only", False), + # GeoNode Apps + GEONODE_APPS_ENABLE=getattr(settings, "GEONODE_APPS_ENABLE", False), + GEONODE_APPS_NAME=getattr(settings, "GEONODE_APPS_NAME", "Apps"), + GEONODE_APPS_NAV_MENU_ENABLE=getattr(settings, "GEONODE_APPS_NAV_MENU_ENABLE", False), + CATALOG_METADATA_TEMPLATE=getattr(settings, "CATALOG_METADATA_TEMPLATE", "catalogue/full_metadata.xml"), + UI_REQUIRED_FIELDS=getattr(settings, "UI_REQUIRED_FIELDS", []), + REQ_THESAURI=[ + f"tkeywords-{x.id}" + for x in Thesaurus.objects.all() + if (x.card_max == -1 and x.card_min == 1) or (x.card_max == 1 and x.card_min == 1) + ], + ADVANCED_EDIT_EXCLUDE_FIELD=getattr(settings, "ADVANCED_EDIT_EXCLUDE_FIELD", []), + PROFILE_EDIT_EXCLUDE_FIELD=getattr(settings, "PROFILE_EDIT_EXCLUDE_FIELD", []), + AVAILABLE_SOCIAL_APPS_COUNT=SocialApp.objects.count(), + GEONODE_APPS_TYPES=get_geonode_app_types(), + ) + return defaults diff --git a/geonode/custom_translations.py b/geonode/custom_translations.py new file mode 100644 index 0000000..f065a8d --- /dev/null +++ b/geonode/custom_translations.py @@ -0,0 +1,123 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +# this file will be used to provide custom translation strings from e.g. fixtures +# it will be parsed by python manage.py makemessages and update the .po files accordingly + +from django.utils.translation import gettext_lazy as _ + +texts = [ + _( + "Flora and/or fauna in natural environment. Examples: wildlife, vegetation, biological sciences, ecology, " + "wilderness, sealife, wetlands, habitat." + ), + _("Biota"), + _("Legal land descriptions. Examples: political and administrative boundaries."), + _("Boundaries"), + _( + "Processes and phenomena of the atmosphere. Examples: cloud cover, weather, climate, atmospheric conditions, " + "climate change, precipitation." + ), + _("Climatology Meteorology Atmosphere"), + _( + "Economic activities, conditions and employment. Examples: production, labour, revenue, commerce, industry, " + "tourism and ecotourism, forestry, fisheries, commercial or subsistence hunting, exploration and exploitation " + "of resources such as minerals, oil and gas." + ), + _("Economy"), + _( + "Height above or below sea level. Examples: altitude, bathymetry, digital elevation models, slope, " + "derived products." + ), + _("Elevation"), + _( + "Environmental resources, protection and conservation. Examples: environmental pollution, waste storage and " + "treatment, environmental impact assessment, monitoring environmental risk, nature reserves, landscape." + ), + _("Environment"), + _( + "Rearing of animals and/or cultivation of plants. Examples: agriculture, irrigation, aquaculture, " + "plantations, herding, pests and diseases affecting crops and livestock." + ), + _("Farming"), + _( + "Information pertaining to earth sciences. Examples: geophysical features and processes, geology, minerals, " + "sciences dealing with the composition, structure and origin of the earth s rocks, risks of earthquakes, " + "volcanic activity, landslides, gravity information, soils, permafrost, hydrogeology, erosion." + ), + _("Geoscientific Information"), + _( + "Health, health services, human ecology, and safety. Examples: disease and illness, factors affecting health, " + "hygiene, substance abuse, mental and physical health, health services." + ), + _("Health"), + _("Base maps. Examples: land cover, topographic maps, imagery, unclassified images, annotations."), + _("Imagery Base Maps Earth Cover"), + _( + "Inland water features, drainage systems and their characteristics. Examples: rivers and glaciers, salt lakes, " + "water utilization plans, dams, currents, floods, water quality, hydrographic charts." + ), + _("Inland Waters"), + _( + "Military bases, structures, activities. Examples: barracks, training grounds, military transportation, " + "information collection." + ), + _("Intelligence Military"), + _( + "Positional information and services. Examples: addresses, geodetic networks, control points, postal zones " + "and services, place names." + ), + _("Location"), + _( + "Features and characteristics of salt water bodies (excluding inland waters). Examples: tides, tidal waves, " + "coastal information, reefs." + ), + _("Oceans"), + _( + "Information used for appropriate actions for future use of the land. Examples: land use maps, zoning maps, " + "cadastral surveys, land ownership." + ), + _("Planning Cadastre"), + _( + "Settlements, anthropology, archaeology, education, traditional beliefs, manners and customs, demographic " + "data, recreational areas and activities, social impact assessments, crime and justice, census information. " + "Economic activities, conditions and employment." + ), + _("Population"), + _( + "Characteristics of society and cultures. Examples: settlements, anthropology, archaeology, education, " + "traditional beliefs, manners and customs, demographic data, recreational areas and activities, social impact " + "assessments, crime and justice, census information." + ), + _("Society"), + _("Man-made construction. Examples: buildings, museums, churches, factories, housing, monuments, shops, towers."), + _("Structure"), + _( + "Means and aids for conveying persons and/or goods. Examples: roads, airports/airstrips, shipping routes, " + "tunnels, nautical charts, vehicle or vessel location, aeronautical charts, railways." + ), + _("Transportation"), + _( + "Energy, water and waste systems and communications infrastructure and services. Examples: hydroelectricity, " + "geothermal, solar and nuclear sources of energy, water purification and distribution, sewage collection and " + "disposal, electricity and gas distribution, data communication, telecommunication, radio, communication " + "networks." + ), + _("Utilities Communication"), +] diff --git a/geonode/decorators.py b/geonode/decorators.py new file mode 100644 index 0000000..6f35616 --- /dev/null +++ b/geonode/decorators.py @@ -0,0 +1,317 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import base64 +import logging + +from functools import wraps + +from django.contrib import auth +from django.conf import settings +from django.http import HttpResponse +from django.contrib.auth import authenticate, login +from django.utils.decorators import classonlymethod +from django.core.exceptions import PermissionDenied + +from geonode.utils import check_ogc_backend, get_client_ip, get_client_host + +logger = logging.getLogger(__name__) + + +def on_ogc_backend(backend_package): + """Decorator for function specific to a certain ogc backend. + + This decorator will wrap function so it only gets executed if the + specified ogc backend is currently used. If not, the function will just + be skipped. + + Useful to decorate features/tests that only available for specific + backend. + """ + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + on_backend = check_ogc_backend(backend_package) + if on_backend: + return func(*args, **kwargs) + + return wrapper + + return decorator + + +def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): + """ + This is a helper function used by both 'logged_in_or_basicauth' and + 'has_perm_or_basicauth' that does the nitty of determining if they + are already logged in or if they have provided proper http-authorization + and returning the view if all goes well, otherwise responding with a 401. + """ + if test_func(request.user): + # Already logged in, just return the view. + # + return view(request, *args, **kwargs) + + # They are not logged in. See if they provided login credentials + # + if "HTTP_AUTHORIZATION" in request.META: + basic_auth = request.META["HTTP_AUTHORIZATION"].split() + if len(basic_auth) == 2: + # NOTE: We are only support basic authentication for now. + # + if basic_auth[0].lower() == "basic": + uname, passwd = base64.b64decode(basic_auth[1]).decode("utf-8").split(":", 1) + user = authenticate(username=uname, password=passwd) + if user and user.is_active: + login(request, user) + request.user = user + auth.update_session_auth_hash(request, user) + return view(request, *args, **kwargs) + + # Either they did not provide an authorization header or + # something in the authorization attempt failed. Send a 401 + # back to them to ask them to authenticate. + # + response = HttpResponse() + response.status_code = 401 + response["WWW-Authenticate"] = f'Basic realm="{realm}"' + return response + + +def view_decorator(fdec, subclass=False): + """ + Change a function decorator into a view decorator. + + https://github.com/lqc/django/tree/cbvdecoration_ticket14512 + """ + + def decorator(cls): + if not hasattr(cls, "as_view"): + raise TypeError("You should only decorate subclasses of View, not mixins.") + if subclass: + cls = type(f"{cls.__name__}WithDecorator({fdec.__name__})", (cls,), {}) + original = cls.as_view.__func__ + + @wraps(original) + def as_view(current, **initkwargs): + return fdec(original(current, **initkwargs)) + + cls.as_view = classonlymethod(as_view) + return cls + + return decorator + + +def view_or_apiauth(view, request, test_func, *args, **kwargs): + """ + This is a helper function used by both 'logged_in_or_basicauth' and + 'has_perm_or_basicauth' that does the nitty of determining if they + are already logged in or if they have provided proper http-authorization + and returning the view if all goes well, otherwise responding with a 401. + """ + if test_func(auth.get_user(request)) or not settings.OAUTH2_API_KEY: + # Already logged in, just return the view. + # + return view(request, *args, **kwargs) + + # They are not logged in. See if they provided login credentials + # + if "HTTP_AUTHORIZATION" in request.META: + _auth = request.META["HTTP_AUTHORIZATION"].split() + if len(_auth) == 2: + # NOTE: We are only support basic authentication for now. + # + if _auth[0].lower() == "apikey": + auth_api_key = _auth[1] + if auth_api_key and auth_api_key == settings.OAUTH2_API_KEY: + return view(request, *args, **kwargs) + + # Either they did not provide an authorization header or + # something in the authorization attempt failed. Send a 401 + # back to them to ask them to authenticate. + # + response = HttpResponse() + response.status_code = 401 + return response + + +def has_perm_or_basicauth(perm, realm=""): + """ + This is similar to the above decorator 'logged_in_or_basicauth' + except that it requires the logged in user to have a specific + permission. + + Use: + + @logged_in_or_basicauth('asforums.view_forumcollection') + def your_view: + ... + + """ + + def view_decorator(func): + def wrapper(request, *args, **kwargs): + return view_or_basicauth(func, request, lambda u: u.has_perm(perm), realm, *args, **kwargs) + + return wrapper + + return view_decorator + + +def superuser_only(function): + """ + Limit view to superusers only. + + Usage: + -------------------------------------------------------------------------- + @superuser_only + def my_view(request): + ... + -------------------------------------------------------------------------- + + or in urls: + + -------------------------------------------------------------------------- + urlpatterns = patterns('', + (r'^foobar/(.*)', is_staff(my_view)), + ) + -------------------------------------------------------------------------- + """ + + def _inner(request, *args, **kwargs): + if not auth.get_user(request).is_superuser and not auth.get_user(request).is_staff: + raise PermissionDenied + return function(request, *args, **kwargs) + + return _inner + + +def check_keyword_write_perms(function): + def _inner(request, *args, **kwargs): + keyword_readonly = ( + settings.FREETEXT_KEYWORDS_READONLY and request.method == "POST" and not auth.get_user(request).is_superuser + ) + request.keyword_readonly = keyword_readonly + if keyword_readonly and "resource-keywords" in request.POST: + return HttpResponse( + "Unauthorized: Cannot edit/create Free-text Keywords", status=401, content_type="application/json" + ) + return function(request, *args, **kwargs) + + return _inner + + +def superuser_protected(function): + """Decorator that forces a view to be accessible by SUPERUSERS only.""" + + def _inner(request, *args, **kwargs): + if not auth.get_user(request).is_superuser: + return HttpResponse( + json.dumps({"error": "unauthorized_request"}), status=403, content_type="application/json" + ) + return function(request, *args, **kwargs) + + return _inner + + +def whitelist_protected(function): + """Decorator that forces a view to be accessible by WHITE_LISTED + IPs only. + """ + + def _inner(request, *args, **kwargs): + if not settings.AUTH_IP_WHITELIST or ( + get_client_ip(request) not in settings.AUTH_IP_WHITELIST + and get_client_host(request) not in settings.AUTH_IP_WHITELIST + ): + return HttpResponse( + json.dumps({"error": "unauthorized_request"}), status=403, content_type="application/json" + ) + return function(request, *args, **kwargs) + + return _inner + + +def logged_in_or_basicauth(realm=""): + """ + A simple decorator that requires a user to be logged in. If they are not + logged in the request is examined for a 'authorization' header. + + If the header is present it is tested for basic authentication and + the user is logged in with the provided credentials. + + If the header is not present a http 401 is sent back to the + requestor to provide credentials. + + The purpose of this is that in several django projects I have needed + several specific views that need to support basic authentication, yet the + web site as a whole used django's provided authentication. + + The uses for this are for urls that are access programmatically such as + by rss feed readers, yet the view requires a user to be logged in. Many rss + readers support supplying the authentication credentials via http basic + auth (and they do NOT support a redirect to a form where they post a + username/password.) + + Use is simple: + + @logged_in_or_basicauth() + def your_view: + ... + + You can provide the name of the realm to ask for authentication within. + """ + + def view_decorator(func): + def wrapper(request, *args, **kwargs): + return view_or_basicauth(func, request, lambda u: u.is_authenticated, realm, *args, **kwargs) + + return wrapper + + return view_decorator + + +def logged_in_or_apiauth(): + def view_decorator(func): + def wrapper(request, *args, **kwargs): + return view_or_apiauth(func, request, lambda u: u.is_authenticated, *args, **kwargs) + + return wrapper + + return view_decorator + + +def superuser_or_apiauth(): + def view_decorator(func): + def wrapper(request, *args, **kwargs): + return view_or_apiauth(func, request, lambda u: u.is_superuser, *args, **kwargs) + + return wrapper + + return view_decorator + + +def dump_func_name(func): + def echo_func(*func_args, **func_kwargs): + logger.debug(f"Start func: {func.__name__}") + return func(*func_args, **func_kwargs) + + return echo_func diff --git a/geonode/documents/__init__.py b/geonode/documents/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/documents/admin.py b/geonode/documents/admin.py new file mode 100644 index 0000000..b571202 --- /dev/null +++ b/geonode/documents/admin.py @@ -0,0 +1,85 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib import admin + +from modeltranslation.admin import TabbedTranslationAdmin + +from geonode.documents.models import Document +from geonode.base.admin import ResourceBaseAdminForm +from geonode.base.admin import metadata_batch_edit + + +class DocumentAdminForm(ResourceBaseAdminForm): + class Meta(ResourceBaseAdminForm.Meta): + model = Document + fields = "__all__" + # exclude = ( + # 'resource', + # ) + + +class DocumentAdmin(TabbedTranslationAdmin): + exclude = ("ll_bbox_polygon", "bbox_polygon", "srid") + list_display = ( + "id", + "title", + "date", + "category", + "group", + "is_approved", + "is_published", + "advertised", + "metadata_completeness", + ) + list_display_links = ("id",) + list_editable = ("title", "category", "group", "is_approved", "is_published", "advertised") + list_filter = ( + "date", + "date_type", + "restriction_code_type", + "category", + "group", + "is_approved", + "is_published", + ) + search_fields = ( + "title", + "abstract", + "purpose", + "is_approved", + "is_published", + ) + readonly_fields = ("geographic_bounding_box",) + date_hierarchy = "date" + form = DocumentAdminForm + actions = [metadata_batch_edit] + + def delete_queryset(self, request, queryset): + """ + We need to invoke the 'ResourceBase.delete' method even when deleting + through the admin batch action + """ + for obj in queryset: + from geonode.resource.manager import resource_manager + + resource_manager.delete(obj.uuid, instance=obj) + + +admin.site.register(Document, DocumentAdmin) diff --git a/geonode/documents/api/__init__.py b/geonode/documents/api/__init__.py new file mode 100644 index 0000000..0044807 --- /dev/null +++ b/geonode/documents/api/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/documents/api/exceptions.py b/geonode/documents/api/exceptions.py new file mode 100644 index 0000000..4850255 --- /dev/null +++ b/geonode/documents/api/exceptions.py @@ -0,0 +1,26 @@ +######################################################################### +# +# Copyright (C) 2022 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from rest_framework.exceptions import APIException + + +class DocumentException(APIException): + status_code = 400 + default_detail = "invalid document" + default_code = "document_exception" + category = "document_api" diff --git a/geonode/documents/api/permissions.py b/geonode/documents/api/permissions.py new file mode 100644 index 0000000..ef83040 --- /dev/null +++ b/geonode/documents/api/permissions.py @@ -0,0 +1,59 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.conf import settings +from rest_framework.filters import BaseFilterBackend + + +class DocumentPermissionsFilter(BaseFilterBackend): + """ + A filter backend that limits results to those where the requesting user + has read object level permissions. + """ + + shortcut_kwargs = { + "accept_global_perms": True, + } + + def filter_queryset(self, request, queryset, view): + # We want to defer this import until runtime, rather than import-time. + # See https://github.com/encode/django-rest-framework/issues/4608 + # (Also see #1624 for why we need to make this import explicitly) + from guardian.shortcuts import get_objects_for_user + from geonode.security.utils import get_visible_resources + + user = request.user + # perm_format = '%(app_label)s.view_%(model_name)s' + # permission = self.perm_format % { + # 'app_label': queryset.model._meta.app_label, + # 'model_name': queryset.model._meta.model_name, + # } + + resources = get_objects_for_user(user, "base.view_resourcebase", **self.shortcut_kwargs).filter( + polymorphic_ctype__model="document" + ) + + obj_with_perms = get_visible_resources( + resources, + user, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ) + + return queryset.filter(id__in=obj_with_perms.values("id")) diff --git a/geonode/documents/api/serializers.py b/geonode/documents/api/serializers.py new file mode 100644 index 0000000..b0c6387 --- /dev/null +++ b/geonode/documents/api/serializers.py @@ -0,0 +1,63 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from dynamic_rest.fields.fields import DynamicComputedField +from rest_framework import serializers +from geonode.base.api.serializers import ResourceBaseSerializer +from geonode.documents.models import Document + +logger = logging.getLogger(__name__) + + +class GeonodeFilePathField(DynamicComputedField): + def get_attribute(self, instance): + return instance.files + + +class DocumentFieldField(DynamicComputedField): + def get_attribute(self, instance): + return instance.files + + +class DocumentSerializer(ResourceBaseSerializer): + title = serializers.CharField(required=False) + file_path = GeonodeFilePathField(required=False, write_only=True) + doc_file = DocumentFieldField(required=False, write_only=True) + + class Meta: + model = Document + name = "document" + view_name = "documents-list" + fields = list( + set( + ResourceBaseSerializer.Meta.fields + + ( + "uuid", + "name", + "href", + "subtype", + "extension", + "mime_type", + "file_path", + "doc_file", + "doc_url", + ) + ) + ) diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py new file mode 100644 index 0000000..4d14474 --- /dev/null +++ b/geonode/documents/api/tests.py @@ -0,0 +1,591 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.contrib.auth import get_user_model +from urllib.parse import urljoin + +from django.urls import reverse +from rest_framework.test import APITransactionTestCase + +from guardian.shortcuts import assign_perm, get_anonymous_user +from geonode import settings + +from geonode.base.populate_test_data import create_models +from geonode.base.enumerations import SOURCE_TYPE_REMOTE +from geonode.documents.models import Document + +logger = logging.getLogger(__name__) + + +class DocumentsApiTests(APITransactionTestCase): + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + def setUp(self): + create_models(b"document") + create_models(b"map") + create_models(b"dataset") + self.admin = get_user_model().objects.get(username="admin") + self.url = reverse("documents-list") + self.invalid_file_path = f"{settings.PROJECT_ROOT}/tests/data/thesaurus.rdf" + self.valid_file_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" + self.no_title_file_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_sld.sld" + + def test_documents(self): + """ + Ensure we can access the Documents list. + """ + url = reverse("documents-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 9) + # Pagination + self.assertEqual(len(response.data["documents"]), 9) + logger.debug(response.data) + + for _l in response.data["documents"]: + self.assertTrue(_l["resource_type"], "document") + + # Get Linked Resources List + resource = Document.objects.first() + + url = urljoin(f"{reverse('documents-detail', kwargs={'pk': resource.pk})}/", "linked_resources/") + assign_perm("base.view_resourcebase", get_anonymous_user(), resource.get_self_resource()) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + layers_data = response.data + self.assertIsNotNone(layers_data) + + # import json + # logger.error(f"{json.dumps(layers_data)}") + + def test_extra_metadata_included_with_param(self): + resource = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{resource.pk}") + data = {"include[]": "metadata"} + + response = self.client.get(url, format="json", data=data) + self.assertIsNotNone(response.data["document"].get("metadata")) + + response = self.client.get(url, format="json") + self.assertNotIn("metadata", response.data["document"]) + + def test_creation_return_error_if_file_is_not_passed(self): + """ + If file_path is not available, should raise error + """ + self.client.force_login(self.admin) + payload = {"document": {"title": "New document", "metadata_only": True}} + expected = { + "success": False, + "errors": ["A file, file path or URL must be speficied"], + "code": "document_exception", + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(400, actual.status_code) + self.assertDictEqual(expected, actual.json()) + + def test_creation_return_error_if_file_is_none(self): + """ + If file_path is not available, should raise error + """ + self.client.force_login(self.admin) + payload = {"document": {"title": "New document", "metadata_only": True, "file_path": None, "doc_file": None}} + expected = { + "success": False, + "errors": ["A file, file path or URL must be speficied"], + "code": "document_exception", + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(400, actual.status_code) + self.assertDictEqual(expected, actual.json()) + + def test_creation_should_rase_exec_for_unsupported_files(self): + self.client.force_login(self.admin) + payload = {"document": {"title": "New document", "metadata_only": True, "file_path": self.invalid_file_path}} + expected = { + "success": False, + "errors": ["The file provided is not in the supported extensions list"], + "code": "document_exception", + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(400, actual.status_code) + self.assertDictEqual(expected, actual.json()) + + def test_document_listing_advertised(self): + document = Document.objects.first() + document.advertised = False + document.save() + + url = reverse("documents-list") + + payload = self.client.get(url) + + prev_count = payload.json().get("total") + # the user can see only the advertised resources + self.assertTrue(Document.objects.count() > prev_count) + + payload = self.client.get(f"{url}?advertised=True") + # so if advertised is True, we dont see the advertised=False resource + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count) + + payload = self.client.get(f"{url}?advertised=False") + # so if advertised is False, we see only the resource with advertised==False + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, 1) + + # if all is requested, we will see all the resources + payload = self.client.get(f"{url}?advertised=all") + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count + 1) + + Document.objects.update(advertised=True) + + def test_creation_should_create_the_doc(self): + """ + If file_path is not available, should raise error + """ + self.client.force_login(self.admin) + payload = { + "document": {"title": "New document for testing", "metadata_only": True, "file_path": self.valid_file_path} + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(201, actual.status_code) + extension = actual.json().get("document", {}).get("extension", "") + self.assertEqual("xml", extension) + self.assertTrue(Document.objects.filter(title="New document for testing").exists()) + + def test_uploading_doc_without_title(self): + """ + A document should be uploaded without specifying a title + """ + self.client.force_login(self.admin) + payload = {"document": {"metadata_only": True, "file_path": self.no_title_file_path}} + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(201, actual.status_code) + extension = actual.json().get("document", {}).get("extension", "") + self.assertEqual("sld", extension) + self.assertTrue(Document.objects.filter(title="test_sld.sld").exists()) + + def test_patch_point_of_contact(self): + document = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{document.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"poc": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [poc.get("pk") for poc in response.json().get("document").get("poc")] for user_id in user_ids + ) + ) + # Resetting all point of contact + response = self.client.patch(url, data={"poc": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [poc.get("pk") for poc in response.json().get("document").get("poc")] + for user_id in user_ids + ) + ) + + def test_patch_metadata_author(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"metadata_author": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [ + metadata_author.get("pk") + for metadata_author in response.json().get("document").get("metadata_author") + ] + for user_id in user_ids + ) + ) + # Resetting all metadata authors + response = self.client.patch(url, data={"metadata_author": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + metadata_author.get("pk") + for metadata_author in response.json().get("document").get("metadata_author") + ] + for user_id in user_ids + ) + ) + + def test_patch_processor(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"processor": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + # check if all set processors are in the return json + self.assertTrue( + all( + user_id in [processor.get("pk") for processor in response.json().get("document").get("processor")] + for user_id in user_ids + ) + ) + # Resetting all processors + response = self.client.patch(url, data={"processor": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [processor.get("pk") for processor in response.json().get("document").get("processor")] + for user_id in user_ids + ) + ) + + def test_patch_publisher(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"publisher": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [publisher.get("pk") for publisher in response.json().get("document").get("publisher")] + for user_id in user_ids + ) + ) + # Resetting all publishers + response = self.client.patch(url, data={"publisher": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [publisher.get("pk") for publisher in response.json().get("document").get("publisher")] + for user_id in user_ids + ) + ) + + def test_patch_custodian(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"custodian": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [custodian.get("pk") for custodian in response.json().get("document").get("custodian")] + for user_id in user_ids + ) + ) + # Resetting all custodians + response = self.client.patch(url, data={"custodian": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [custodian.get("pk") for custodian in response.json().get("document").get("custodian")] + for user_id in user_ids + ) + ) + + def test_patch_distributor(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"distributor": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [distributor.get("pk") for distributor in response.json().get("document").get("distributor")] + for user_id in user_ids + ) + ) + # Resetting all distributers + response = self.client.patch(url, data={"distributor": []}, format="json") + + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [distributor.get("pk") for distributor in response.json().get("document").get("distributor")] + for user_id in user_ids + ) + ) + + def test_patch_resource_user(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"resource_user": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [resource_user.get("pk") for resource_user in response.json().get("document").get("resource_user")] + for user_id in user_ids + ) + ) + # Resetting all resource users + response = self.client.patch(url, data={"resource_user": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + resource_user.get("pk") for resource_user in response.json().get("document").get("resource_user") + ] + for user_id in user_ids + ) + ) + + def test_patch_resource_provider(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"resource_provider": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [ + resource_provider.get("pk") + for resource_provider in response.json().get("document").get("resource_provider") + ] + for user_id in user_ids + ) + ) + # Resetting all principal investigator + response = self.client.patch(url, data={"resource_provider": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + resource_provider.get("pk") + for resource_provider in response.json().get("document").get("resource_provider") + ] + for user_id in user_ids + ) + ) + + def test_patch_originator(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"originator": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [originator.get("pk") for originator in response.json().get("document").get("originator")] + for user_id in user_ids + ) + ) + # Resetting all originators + response = self.client.patch(url, data={"originator": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [originator.get("pk") for originator in response.json().get("document").get("originator")] + for user_id in user_ids + ) + ) + + def test_patch_principal_investigator(self): + layer = Document.objects.first() + url = urljoin(f"{reverse('documents-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"principal_investigator": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [ + principal_investigator.get("pk") + for principal_investigator in response.json().get("document").get("principal_investigator") + ] + for user_id in user_ids + ) + ) + # Resetting all principal investigator + response = self.client.patch(url, data={"principal_investigator": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + principal_investigator.get("pk") + for principal_investigator in response.json().get("document").get("principal_investigator") + ] + for user_id in user_ids + ) + ) + + def test_creation_should_create_the_doc_and_update_the_bbox(self): + """ + If file_path is not available, should raise error + """ + self.client.force_login(self.admin) + payload = { + "document": { + "title": "New document for testing", + "metadata_only": True, + "file_path": self.valid_file_path, + "extent": {"coords": [1123692.0, 5338214.0, 1339852.0, 5482615.0], "srid": "EPSG:3857"}, + }, + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(201, actual.status_code) + extension = actual.json().get("document", {}).get("extension", "") + self.assertEqual("xml", extension) + doc = Document.objects.filter(title="New document for testing").all() + self.assertTrue(doc.exists()) + x = doc.first() + x.refresh_from_db() + self.assertEqual("EPSG:3857", x.srid) + self.assertEqual(actual.json()["document"].get("extent")["srid"], "EPSG:4326") + self.assertEqual( + actual.json()["document"].get("extent")["coords"], + [10.094296982428332, 43.1721654049465, 12.03609530058109, 44.11086592050112], + ) + + def test_file_path_and_doc_path_are_not_returned(self): + """ + If file_path and doc_path should not be visible + from the GET payload + """ + actual = self.client.get(self.url) + self.assertEqual(200, actual.status_code) + _doc_payload = actual.json().get("document", {}) + self.assertFalse("file_path" in _doc_payload) + self.assertFalse("doc_path" in _doc_payload) + + def test_creation_from_url_should_create_the_doc(self): + """ + If file_path is not available, should raise error + """ + self.client.force_login(self.admin) + doc_url = "https://example.com/image" + payload = { + "document": { + "title": "New document from URL for testing", + "metadata_only": False, + "doc_url": doc_url, + "extension": "jpeg", + } + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(201, actual.status_code) + created_doc_url = actual.json().get("document", {}).get("doc_url", "") + self.assertEqual(created_doc_url, doc_url) + + def test_remote_document_is_marked_remote(self): + """Tests creating an external document set its sourcetype to REMOTE.""" + self.client.force_login(self.admin) + doc_url = "https://example.com/image" + payload = { + "document": { + "title": "A remote document is remote", + "doc_url": doc_url, + "extension": "jpeg", + } + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(201, actual.status_code) + created_sourcetype = actual.json().get("document", {}).get("sourcetype", "") + self.assertEqual(created_sourcetype, SOURCE_TYPE_REMOTE) + + def test_either_path_or_url_doc(self): + """ + If file_path is not available, should raise error + """ + self.client.force_login(self.admin) + doc_url = "https://example.com/image" + payload = { + "document": { + "title": "New document from URL for testing", + "metadata_only": False, + "doc_url": doc_url, + "file_path": self.valid_file_path, + "extension": "jpeg", + } + } + actual = self.client.post(self.url, data=payload, format="json") + expected = { + "success": False, + "errors": ["Either a file or a URL must be specified, not both"], + "code": "document_exception", + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(400, actual.status_code) + self.assertDictEqual(expected, actual.json()) diff --git a/geonode/documents/api/urls.py b/geonode/documents/api/urls.py new file mode 100644 index 0000000..91a51d5 --- /dev/null +++ b/geonode/documents/api/urls.py @@ -0,0 +1,25 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.api.urls import router + +from . import views + +router.register(r"documents", views.DocumentViewSet, "documents") + +urlpatterns = [] diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py new file mode 100644 index 0000000..d68ca3a --- /dev/null +++ b/geonode/documents/api/views.py @@ -0,0 +1,157 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from drf_spectacular.utils import extend_schema +from pathlib import Path +from dynamic_rest.viewsets import DynamicModelViewSet +from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter + +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.authentication import SessionAuthentication, BasicAuthentication +from oauth2_provider.contrib.rest_framework import OAuth2Authentication +from geonode import settings + +from geonode.assets.utils import create_asset_and_link +from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter +from geonode.base.api.mixins import AdvertisedListMixin +from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.base.api.permissions import UserHasPerms +from geonode.base.api.views import base_linked_resources, ApiPresetsInitializer +from geonode.base import enumerations +from geonode.documents.api.exceptions import DocumentException +from geonode.documents.models import Document + +from geonode.base.api.serializers import ResourceBaseSerializer +from geonode.resource.utils import resourcebase_post_save +from geonode.storage.manager import StorageManager +from geonode.resource.manager import resource_manager + +from .serializers import DocumentSerializer +from .permissions import DocumentPermissionsFilter + +import logging + + +logger = logging.getLogger(__name__) + + +class DocumentViewSet(ApiPresetsInitializer, DynamicModelViewSet, AdvertisedListMixin): + """ + API endpoint that allows documents to be viewed or edited. + """ + + http_method_names = ["get", "patch", "put", "post"] + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [ + IsAuthenticatedOrReadOnly, + UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}), + ] + filter_backends = [ + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + ExtentFilter, + DocumentPermissionsFilter, + ] + queryset = Document.objects.all().order_by("-created") + serializer_class = DocumentSerializer + pagination_class = GeoNodeApiPagination + + def perform_create(self, serializer): + """ + Function to create document via API v2. + file_path: path to the file + doc_file: the open file + + The API expect this kind of JSON: + { + "document": { + "title": "New document", + "metadata_only": true, + "file_path": "/home/mattia/example.json" + } + } + File path rappresent the filepath where the file to upload is saved. + + or can be also a form-data: + curl --location --request POST 'http://localhost:8000/api/v2/documents' \ + --form 'title="Super Title2"' \ + --form 'doc_file=@"/C:/Users/user/Pictures/BcMc-a6T9IM.jpg"' \ + --form 'metadata_only="False"' + """ + manager = None + serializer.is_valid(raise_exception=True) + file = serializer.validated_data.pop("file_path", None) or serializer.validated_data.pop("doc_file", None) + doc_url = serializer.validated_data.pop("doc_url", None) + extension = serializer.validated_data.pop("extension", None) + + if not file and not doc_url: + raise DocumentException(detail="A file, file path or URL must be speficied") + + if file and doc_url: + raise DocumentException(detail="Either a file or a URL must be specified, not both") + + if not extension: + filename = file if isinstance(file, str) else file.name + extension = Path(filename).suffix.replace(".", "") + + if extension not in settings.ALLOWED_DOCUMENT_TYPES: + raise DocumentException("The file provided is not in the supported extensions list") + + try: + payload = { + "owner": self.request.user, + "extension": extension, + "resource_type": "document", + } + if doc_url: + payload["doc_url"] = doc_url + payload["sourcetype"] = enumerations.SOURCE_TYPE_REMOTE + + resource = serializer.save(**payload) + + if file: + manager = StorageManager(remote_files={"base_file": file}) + manager.clone_remote_files() + create_asset_and_link( + resource, self.request.user, [manager.get_retrieved_paths().get("base_file")], clone_files=True + ) + manager.delete_retrieved_paths(force=True) + + resource.set_missing_info() + resourcebase_post_save(resource.get_real_instance()) + resource_manager.set_permissions(None, instance=resource, permissions=None, created=True) + resource.handle_moderated_uploads() + resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=False) + return resource + except Exception as e: + logger.error(f"Error creating document {serializer.validated_data}", exc_info=e) + if manager: + manager.delete_retrieved_paths() + raise e + + @extend_schema( + methods=["get"], + responses={200: ResourceBaseSerializer(many=True)}, + description="API endpoint allowing to retrieve linked resources", + ) + @action(detail=True, methods=["get"]) + def linked_resources(self, request, pk=None, *args, **kwargs): + return base_linked_resources(self.get_object().get_real_instance(), request.user, request.GET) diff --git a/geonode/documents/apps.py b/geonode/documents/apps.py new file mode 100644 index 0000000..72e2ad9 --- /dev/null +++ b/geonode/documents/apps.py @@ -0,0 +1,58 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.utils.translation import gettext_noop as _ +from geonode.notifications_helper import NotificationsAppConfigBase +from django.apps import AppConfig + + +class DocumentsAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.documents" + NOTIFICATIONS = ( + ( + "document_created", + _("Document Created"), + _("A Document was created"), + ), + ( + "document_updated", + _("Document Updated"), + _("A Document was updated"), + ), + ( + "document_approved", + _("Document Approved"), + _("A Document was approved by a Manager"), + ), + ( + "document_published", + _("Document Published"), + _("A Document was published"), + ), + ( + "document_deleted", + _("Document Deleted"), + _("A Document was deleted"), + ), + ( + "document_rated", + _("Rating for Document"), + _("A rating was given to a document"), + ), + ) diff --git a/geonode/documents/enumerations.py b/geonode/documents/enumerations.py new file mode 100644 index 0000000..8b270b7 --- /dev/null +++ b/geonode/documents/enumerations.py @@ -0,0 +1,134 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +# DOCUMENT_TYPE_MAP and DOCUMENT_MIMETYPE_MAP +# match values in settings.ALLOWED_DOCUMENT_TYPES + +DOCUMENT_TYPE_MAP = { + "txt": "text", + "log": "text", + "doc": "text", + "docx": "text", + "ods": "text", + "odt": "text", + "sld": "text", + "qml": "text", + "xls": "text", + "xlsx": "text", + "xml": "text", + "bm": "image", + "bmp": "image", + "dwg": "archive", + "dxf": "archive", + "fif": "image", + "gif": "image", + "jpg": "image", + "jpe": "image", + "jpeg": "image", + "png": "image", + "tif": "archive", + "tiff": "archive", + "pbm": "archive", + "odp": "presentation", + "ppt": "presentation", + "pptx": "presentation", + "pdf": "presentation", + "tar": "archive", + "tgz": "archive", + "rar": "archive", + "gz": "archive", + "7z": "archive", + "zip": "archive", + "aif": "audio", + "aifc": "audio", + "aiff": "audio", + "au": "audio", + "mp3": "audio", + "mpga": "audio", + "wav": "audio", + "afl": "video", + "avi": "video", + "avs": "video", + "fli": "video", + "mp2": "video", + "mp4": "video", + "mpg": "video", + "ogg": "video", + "webm": "video", + "3gp": "video", + "flv": "video", + "vdo": "video", +} + + +DOCUMENT_MIMETYPE_MAP = { + "txt": "text/plain", + "log": "text/plain", + "doc": "application/msword", + "docx": "application/msword", + "ods": "text/plain", + "odt": "text/plain", + "sld": "text/plain", + "qml": "text/plain", + "xls": "application/excel", + "xlsx": "application/vnd.ms-excel", + "xml": "application/xml", + "bm": "image/bmp", + "bmp": "image/bmp", + "dwg": "image/vnd.dwg", + "dxf": "image/vnd.dwg", + "fif": "image/fif", + "gif": "image/gif", + "jpe": "image/jpeg", + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "tif": "image/tiff", + "tiff": "image/tiff", + "pbm": "image/x-portable-bitmap", + "odp": "application/odp", + "ppt": "application/powerpoint", + "pptx": "application/x-mspowerpoint", + "pdf": "application/pdf", + "tar": "application/x-tar", + "tgz": "application/x-compressed", + "rar": "application/x-rar-compressed", + "gz": "application/x-gzip", + "7z": "application/zip", + "zip": "application/zip", + "aif": "audio/aiff", + "aifc": "audio/aiff", + "aiff": "audio/aiff", + "au": "audio/basic", + "mp3": "audio/mp3", + "mpga": "audio/mpeg", + "wav": "audio/wav", + "afl": "video/animaflex", + "avi": "video/avi", + "avs": "video/avs-video", + "fli": "video/fli", + "mp2": "video/mpeg", + "mp4": "video/mp4", + "mpg": "video/mpeg", + "ogg": "video/ogg", + "webm": "video/webm", + "3gp": "video/3gp", + "flv": "video/flv", + "vdo": "video/vdo", +} diff --git a/geonode/documents/exif/README.md b/geonode/documents/exif/README.md new file mode 100644 index 0000000..3056dd1 --- /dev/null +++ b/geonode/documents/exif/README.md @@ -0,0 +1,11 @@ +# Exit App Plugin + +Exif is a contrib app starting with GeoNode 2.4. +The exif contrib app extracts exif metadata from uploaded JPEG documents. +This contrib app helps automatically fill key metadata gaps, such as the date a picture was taken. +If a document has exif metadata, the exif metadata is visible on a new tab on the document detail page. + +Exif data can contain very valuable metadata such as the date and time a picture was taken, the GPS location, and photometric properties. +Not all this metadata is currently used. + +See the Wikipedia article on Exif at https://en.wikipedia.org/wiki/Exchangeable_image_file_format to learn more. diff --git a/geonode/documents/exif/__init__.py b/geonode/documents/exif/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/documents/exif/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/documents/exif/apps.py b/geonode/documents/exif/apps.py new file mode 100644 index 0000000..eaeb3e2 --- /dev/null +++ b/geonode/documents/exif/apps.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + + +class GeoNodeDocumentsExifAppConfig(AppConfig): + name = "geonode.documents.exif" + verbose_name = "GeoNode Documents Exif" diff --git a/geonode/documents/exif/templates/exif/_exif_document_detail.html b/geonode/documents/exif/templates/exif/_exif_document_detail.html new file mode 100644 index 0000000..5998f59 --- /dev/null +++ b/geonode/documents/exif/templates/exif/_exif_document_detail.html @@ -0,0 +1,24 @@ +{% load i18n %} + +
    +
    +
    {% trans 'Width' %}
    +
    {{ exif_data.width }}
    +
    {% trans 'Height' %}
    +
    {{ exif_data.height }}
    +
    {% trans 'Make' %}
    +
    {{ exif_data.make }}
    +
    {% trans 'Model' %}
    +
    {{ exif_data.model }}
    +
    {% trans 'Date' %}
    +
    {{ exif_data.date }}
    +
    {% trans 'Latitude' %}
    +
    {% if exif_data.lat %}{{ exif_data.lat }}{% else %}{% trans 'Not specified' %}{% endif %}
    +
    {% trans 'Longitude' %}
    +
    {% if exif_data.lon %}{{ exif_data.lon }}{% else %}{% trans 'Not specified' %}{% endif %}
    +
    {% trans 'Flash' %}
    +
    {% if exif_data.flash %}{% trans 'True' %}{% else %}{% trans 'False'%}{% endif %}
    +
    {% trans 'Speed Rating' %}
    +
    {{ exif_data.speed }}
    +
    +
    diff --git a/geonode/documents/exif/tests.py b/geonode/documents/exif/tests.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/documents/exif/tests.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/documents/exif/utils.py b/geonode/documents/exif/utils.py new file mode 100644 index 0000000..2c751e3 --- /dev/null +++ b/geonode/documents/exif/utils.py @@ -0,0 +1,137 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import logging + +from slugify import slugify +from datetime import datetime + +logger = logging.getLogger(__name__) + +ABSTRACT_TEMPLATE_MODEL_DATE_LATLON = "Image shot by {model} on {date} at {lat}, {lon} (latitude, longitude)" +ABSTRACT_TEMPLATE_MODEL_DATE = "Image shot by {model} on {date}" +ABSTRACT_TEMPLATE_MODEL = "Image shot by {model}" +ABSTRACT_TEMPLATE_DATE = "Image shot on {date}" + + +def convertExifDateToDjangoDate(date): + a = list(date.replace(" ", "T")) + a[4] = "-" + a[7] = "-" + a[10] = "T" + + return datetime( + int("".join(a[0:4])), + int("".join(a[5:7])), + int("".join(a[8:10])), + int("".join(a[11:13])), + int("".join(a[14:16])), + ) + + +def convertExifLocationToDecimalDegrees(location, direction): + if location: + dd = 0.0 + d, m, s = location + dd += float(d) + dd += float(m) / 60.0 + dd += float(s) / 3600.0 + + if direction: + if direction.lower() == "s" or direction.lower() == "w": + dd = dd * -1.0 + return dd + else: + return None + + +def exif_extract_metadata_doc(doc): + if not doc: + return None + + if not doc.files: + return None + + _, ext = os.path.splitext(os.path.basename(doc.files[0])) + + if ext[1:] in {"jpg", "jpeg"}: + from PIL import Image, ExifTags + + img = Image.open(doc.doc_file.path) + exif_data = {ExifTags.TAGS[k]: v for k, v in img._getexif().items() if k in ExifTags.TAGS} + + model = None + date = None + keywords = [] + bbox = None + lat = None + lon = None + abstract = None + + if "DateTime" in exif_data: + date = exif_data["DateTime"] + elif "DateTimeOriginal" in exif_data: + date = exif_data["DateTimeOriginal"] + elif "DateTimeDigitized" in exif_data: + date = exif_data["DateTimeDigitized"] + + if date: + try: + date = convertExifDateToDjangoDate(date) + except Exception: + logger.error("Could not parse exif date") + date = None + + if "Make" in exif_data: + keywords.append(slugify(exif_data["Make"])) + + if "Model" in exif_data: + model = exif_data.get("Model", None) + keywords.append(slugify(model)) + + if "GPSInfo" in exif_data: + gpsinfo = {} + for key in exif_data["GPSInfo"].keys(): + decode = ExifTags.GPSTAGS.get(key, key) + gpsinfo[decode] = exif_data["GPSInfo"][key] + if "GPSLatitude" in gpsinfo and "GPSLongitude" in gpsinfo: + lat = convertExifLocationToDecimalDegrees(gpsinfo["GPSLatitude"], gpsinfo.get("GPSLatitudeRef", "N")) + lon = convertExifLocationToDecimalDegrees(gpsinfo["GPSLongitude"], gpsinfo.get("GPSLongitudeRef", "E")) + bbox = [lon, lon, lat, lat] + + abstract = exif_build_abstract(model=model, date=date, lat=lat, lon=lon) + + return {"date": date, "keywords": keywords, "bbox": bbox, "abstract": abstract} + + else: + return None + + +def exif_build_abstract(model=None, date=None, lat=None, lon=None): + if model and date and lat and lon: + return ABSTRACT_TEMPLATE_MODEL_DATE_LATLON.format(model=model, date=date, lat=lat, lon=lon) + elif model and date: + return ABSTRACT_TEMPLATE_MODEL_DATE.format(model=model, date=date) + elif model: + return ABSTRACT_TEMPLATE_MODEL.format(model=model) + elif date: + return ABSTRACT_TEMPLATE_DATE.format(date=date) + else: + return None diff --git a/geonode/documents/forms.py b/geonode/documents/forms.py new file mode 100644 index 0000000..ebd6598 --- /dev/null +++ b/geonode/documents/forms.py @@ -0,0 +1,221 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import json +import logging + +from modeltranslation.forms import TranslationModelForm + +from django import forms +from django.conf import settings +from django.forms import HiddenInput +from django.utils.translation import gettext_lazy as _ +from django.template.defaultfilters import filesizeformat + +from geonode.base.forms import ResourceBaseForm, get_tree_data +from geonode.documents.models import Document +from geonode.upload.models import UploadSizeLimit +from geonode.upload.api.exceptions import FileUploadLimitException + +logger = logging.getLogger(__name__) + + +class SizeRestrictedFileField(forms.FileField): + """ + Same as FileField, but checks file max_size based on the value stored on `field_slug`. + * field_slug - a slug indicating the database object from where the max_size will be retrieved. + """ + + def __init__(self, *args, **kwargs): + self.field_slug = kwargs.pop("field_slug") + super(SizeRestrictedFileField, self).__init__(*args, **kwargs) + + def clean(self, *args, **kwargs): + data = super(SizeRestrictedFileField, self).clean(*args, **kwargs) + if data is None: + return data + file_size = self._get_file_size(data) + if file_size is not None: + # Only query the DB for a max_size when there is a file_size + max_size = self._get_max_size() + # Validate + if file_size is not None and file_size > max_size: + raise FileUploadLimitException( + _(f"File size size exceeds {filesizeformat(max_size)}. Please try again with a smaller file.") + ) + return data + + def _get_file_size(self, data): + try: + file_size = data.size + except AttributeError: + file_size = None + return file_size + + def _get_max_size(self): + try: + max_size_db_obj = UploadSizeLimit.objects.get(slug=self.field_slug) + except UploadSizeLimit.DoesNotExist: + max_size_db_obj = UploadSizeLimit.objects.create_default_limit_with_slug(slug=self.field_slug) + return max_size_db_obj.max_size + + +class DocumentForm(ResourceBaseForm): + title = forms.CharField(required=False) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["regions"].choices = get_tree_data() + for field in self.fields: + help_text = self.fields[field].help_text + self.fields[field].help_text = None + if help_text != "": + self.fields[field].widget.attrs.update( + { + "class": "has-external-popover", + "data-content": help_text, + "placeholder": help_text, + "data-placement": "right", + "data-container": "body", + "data-html": "true", + } + ) + + class Meta(ResourceBaseForm.Meta): + model = Document + exclude = ResourceBaseForm.Meta.exclude + ( + "content_type", + "object_id", + "doc_file", + "extension", + "subtype", + "doc_url", + ) + + +class DocumentDescriptionForm(forms.Form): + title = forms.CharField(max_length=300) + abstract = forms.CharField(max_length=2000, widget=forms.Textarea, required=False) + keywords = forms.CharField(max_length=500, required=False) + + +class DocumentCreateForm(TranslationModelForm): + """ + The document upload form. + """ + + permissions = forms.CharField( + widget=HiddenInput(attrs={"name": "permissions", "id": "permissions"}), required=False + ) + + doc_file = SizeRestrictedFileField(label=_("File"), required=False, field_slug="document_upload_size") + + class Meta: + model = Document + fields = ["title", "doc_file", "doc_url", "extension"] + widgets = { + "name": HiddenInput(attrs={"cols": 80, "rows": 20}), + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def clean_permissions(self): + """ + Ensures the JSON field is JSON. + """ + permissions = self.cleaned_data["permissions"] + + if not self.fields["permissions"].required and (permissions is None or permissions == ""): + return None + + try: + return json.loads(permissions) + except ValueError: + raise forms.ValidationError(_("Permissions must be valid JSON.")) + + def clean(self): + """ + Ensures the doc_file or the doc_url field is populated. + """ + cleaned_data = super().clean() + doc_file = self.cleaned_data.get("doc_file") + doc_url = self.cleaned_data.get("doc_url") + extension = self.cleaned_data.get("extension") + + if not doc_file and not doc_url and "doc_file" not in self.errors and "doc_url" not in self.errors: + logger.error("Document must be a file or url.") + raise forms.ValidationError(_("Document must be a file or url.")) + + if doc_file and doc_url: + logger.error("A document cannot have both a file and a url.") + raise forms.ValidationError(_("A document cannot have both a file and a url.")) + + if extension: + cleaned_data["extension"] = extension.replace(".", "").lower() + + return cleaned_data + + def clean_doc_file(self): + """ + Ensures the doc_file is valid. + """ + doc_file = self.cleaned_data.get("doc_file") + + if doc_file and not os.path.splitext(doc_file.name)[1].lower()[1:] in settings.ALLOWED_DOCUMENT_TYPES: + logger.debug("This file type is not allowed") + raise forms.ValidationError(_("This file type is not allowed")) + + return doc_file + + +class DocumentReplaceForm(forms.ModelForm): + """ + The form used to replace a document. + """ + + doc_file = SizeRestrictedFileField(label=_("File"), required=True, field_slug="document_upload_size") + + class Meta: + model = Document + fields = ["doc_file"] + + def clean(self): + """ + Ensures the doc_file field is populated. + """ + cleaned_data = super().clean() + doc_file = self.cleaned_data.get("doc_file") + + if not doc_file: + raise forms.ValidationError(_("Document must be a file.")) + + return cleaned_data + + def clean_doc_file(self): + """ + Ensures the doc_file is valid. + """ + doc_file = self.cleaned_data.get("doc_file") + + if doc_file and not os.path.splitext(doc_file.name)[1].lower()[1:] in settings.ALLOWED_DOCUMENT_TYPES: + raise forms.ValidationError(_("This file type is not allowed")) + + return doc_file diff --git a/geonode/documents/management/__init__.py b/geonode/documents/management/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/documents/management/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/documents/management/commands/__init__.py b/geonode/documents/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/documents/management/commands/delete_orphaned_files.py b/geonode/documents/management/commands/delete_orphaned_files.py new file mode 100644 index 0000000..7398961 --- /dev/null +++ b/geonode/documents/management/commands/delete_orphaned_files.py @@ -0,0 +1,28 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand +from geonode.documents.utils import delete_orphaned_document_files + + +class Command(BaseCommand): + help = ("Delete orphaned files of already deleted documents.") + + def handle(self, *args, **options): + delete_orphaned_document_files() diff --git a/geonode/documents/management/commands/sync_geonode_documents.py b/geonode/documents/management/commands/sync_geonode_documents.py new file mode 100644 index 0000000..1d8a1be --- /dev/null +++ b/geonode/documents/management/commands/sync_geonode_documents.py @@ -0,0 +1,56 @@ +######################################################################### +# +# Copyright (C) 2022 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.core.management.base import BaseCommand +from geonode.documents.models import Document +from geonode.documents.tasks import create_document_thumbnail + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = ("Update documents. For the moment only thumbnails can be updated") + + def add_arguments(self, parser): + parser.add_argument( + '--updatethumbnails', + action='store_true', + dest="updatethumbnails", + default=False, + help="Generate thumbnails for documents. Only documents without a thumbnail will be considered, unless --force is used.") + + parser.add_argument( + '--force', + action='store_true', + dest="force", + default=False, + help="Force the update of thumbnails for all documents.") + + def handle(self, *args, **options): + updatethumbnails = options.get('updatethumbnails') + force = options.get('force') + for doc in Document.objects.all(): + if updatethumbnails: + if (doc.thumbnail_url is None or doc.thumbnail_url == '') or force: + try: + create_document_thumbnail(doc.id) + except Exception: + logger.error(f"[ERROR] Thumbnail for [{doc.name}] couldn't be created") diff --git a/geonode/documents/migrations/0028_auto_20170801_1228.py b/geonode/documents/migrations/0028_auto_20170801_1228.py new file mode 100644 index 0000000..c77b4b3 --- /dev/null +++ b/geonode/documents/migrations/0028_auto_20170801_1228.py @@ -0,0 +1,31 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '27_drop_resource_columns_from_document_table'), + ] + + operations = [ + migrations.AlterField( + model_name='document', + name='abstract_en', + field=models.TextField(help_text='brief narrative summary of the content of the resource(s)', max_length=2000, null=True, verbose_name='abstract', blank=True), + ), + migrations.AlterField( + model_name='document', + name='data_quality_statement_en', + field=models.TextField(help_text="general explanation of the data producer's knowledge about the lineage of a dataset", max_length=2000, null=True, verbose_name='data quality statement', blank=True), + ), + migrations.AlterField( + model_name='document', + name='purpose_en', + field=models.TextField(help_text='summary of the intentions with which the resource(s) was developed', max_length=500, null=True, verbose_name='purpose', blank=True), + ), + migrations.AlterField( + model_name='document', + name='supplemental_information_en', + field=models.TextField(default='No information provided', max_length=2000, null=True, verbose_name='supplemental information', help_text='any other descriptive information about the dataset'), + ), + ] diff --git a/geonode/documents/migrations/0028_auto_20170801_1228_squashed_0035_auto_20190404_0820.py b/geonode/documents/migrations/0028_auto_20170801_1228_squashed_0035_auto_20190404_0820.py new file mode 100644 index 0000000..79eb762 --- /dev/null +++ b/geonode/documents/migrations/0028_auto_20170801_1228_squashed_0035_auto_20190404_0820.py @@ -0,0 +1,64 @@ +# Generated by Django 1.11.20 on 2019-04-04 08:24 + + +from django.db import migrations, models +import django.db.models.manager + + +class Migration(migrations.Migration): + + replaces = [('documents', '0028_auto_20170801_1228'), ('documents', '0029_auto_20180301_1947'), ('documents', '0030_auto_20180302_0430'), ('documents', '0031_auto_20180409_1238'), ('documents', '0032_auto_20180412_0822'), ('documents', '0033_auto_20180414_2120'), ('documents', '0034_auto_20190329_1652'), ('documents', '0035_auto_20190404_0820')] + + dependencies = [ + ('documents', '27_drop_resource_columns_from_document_table'), + ] + + operations = [ + migrations.AlterField( + model_name='document', + name='abstract_en', + field=models.TextField(blank=True, help_text='brief narrative summary of the content of the resource(s)', max_length=2000, null=True, verbose_name='abstract'), + ), + migrations.AlterField( + model_name='document', + name='data_quality_statement_en', + field=models.TextField(blank=True, help_text="general explanation of the data producer's knowledge about the lineage of a dataset", max_length=2000, null=True, verbose_name='data quality statement'), + ), + migrations.AlterField( + model_name='document', + name='purpose_en', + field=models.TextField(blank=True, help_text='summary of the intentions with which the resource(s) was developed', max_length=500, null=True, verbose_name='purpose'), + ), + migrations.AlterField( + model_name='document', + name='supplemental_information_en', + field=models.TextField(default='No information provided', help_text='any other descriptive information about the dataset', max_length=2000, null=True, verbose_name='supplemental information'), + ), + migrations.AlterModelManagers( + name='document', + managers=[ + ('objects', django.db.models.manager.Manager()), + ('base_objects', django.db.models.manager.Manager()), + ], + ), + migrations.AlterModelOptions( + name='document', + options={'base_manager_name': 'objects'}, + ), + migrations.AlterModelManagers( + name='document', + managers=[ + ], + ), + migrations.AlterModelOptions( + name='document', + options={}, + ), + migrations.AlterModelManagers( + name='document', + managers=[ + ('objects', django.db.models.manager.Manager()), + ('base_objects', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/geonode/documents/migrations/0029_auto_20180301_1947.py b/geonode/documents/migrations/0029_auto_20180301_1947.py new file mode 100644 index 0000000..9c25406 --- /dev/null +++ b/geonode/documents/migrations/0029_auto_20180301_1947.py @@ -0,0 +1,22 @@ +# Generated by Django 1.11.10 on 2018-03-02 01:47 + + +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0028_auto_20170801_1228'), + ] + + operations = [ + migrations.AlterModelManagers( + name='document', + managers=[ + ('objects', django.db.models.manager.Manager()), + ('base_objects', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/geonode/documents/migrations/0029_auto_20190429_0831.py b/geonode/documents/migrations/0029_auto_20190429_0831.py new file mode 100644 index 0000000..3cea3f2 --- /dev/null +++ b/geonode/documents/migrations/0029_auto_20190429_0831.py @@ -0,0 +1,23 @@ +# Generated by Django 1.11.20 on 2019-04-29 08:31 + + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0028_auto_20170801_1228_squashed_0035_auto_20190404_0820'), + ] + + operations = [ + migrations.AlterModelOptions( + name='document', + options={'base_manager_name': 'objects'}, + ), + migrations.AlterModelManagers( + name='document', + managers=[ + ], + ), + ] diff --git a/geonode/documents/migrations/0030_auto_20180302_0430.py b/geonode/documents/migrations/0030_auto_20180302_0430.py new file mode 100644 index 0000000..7c45dec --- /dev/null +++ b/geonode/documents/migrations/0030_auto_20180302_0430.py @@ -0,0 +1,19 @@ +# Generated by Django 1.9.13 on 2018-03-02 10:30 + + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0029_auto_20180301_1947'), + ] + + operations = [ + migrations.AlterModelManagers( + name='document', + managers=[ + ], + ), + ] diff --git a/geonode/documents/migrations/0030_auto_20200305_1520.py b/geonode/documents/migrations/0030_auto_20200305_1520.py new file mode 100644 index 0000000..1e01776 --- /dev/null +++ b/geonode/documents/migrations/0030_auto_20200305_1520.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.10 on 2020-03-05 15:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0029_auto_20190429_0831'), + ] + + operations = [ + migrations.AlterField( + model_name='documentresourcelink', + name='content_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType'), + ), + migrations.AlterField( + model_name='documentresourcelink', + name='document', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='links', to='documents.Document'), + ), + ] diff --git a/geonode/documents/migrations/0031_auto_20180409_1238.py b/geonode/documents/migrations/0031_auto_20180409_1238.py new file mode 100644 index 0000000..439489c --- /dev/null +++ b/geonode/documents/migrations/0031_auto_20180409_1238.py @@ -0,0 +1,22 @@ +# Generated by Django 1.11.11 on 2018-04-09 12:38 + + +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0030_auto_20180302_0430'), + ] + + operations = [ + migrations.AlterModelManagers( + name='document', + managers=[ + ('objects', django.db.models.manager.Manager()), + ('base_objects', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/geonode/documents/migrations/0031_auto_20201107_2241.py b/geonode/documents/migrations/0031_auto_20201107_2241.py new file mode 100644 index 0000000..46193d2 --- /dev/null +++ b/geonode/documents/migrations/0031_auto_20201107_2241.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.16 on 2020-11-07 22:41 + +from django.db import migrations, models +import uuid_upload_path.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0030_auto_20200305_1520'), + ] + + operations = [ + migrations.AlterField( + model_name='document', + name='doc_file', + field=models.FileField(blank=True, max_length=255, null=True, upload_to=uuid_upload_path.storage.upload_to, verbose_name='File'), + ), + ] diff --git a/geonode/documents/migrations/0032_auto_20180412_0822.py b/geonode/documents/migrations/0032_auto_20180412_0822.py new file mode 100644 index 0000000..1b09621 --- /dev/null +++ b/geonode/documents/migrations/0032_auto_20180412_0822.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0031_auto_20180409_1238'), + ] + + operations = [ + migrations.AlterModelManagers( + name='document', + managers=[ + ], + ), + ] diff --git a/geonode/documents/migrations/0032_remove_document_doc_file.py b/geonode/documents/migrations/0032_remove_document_doc_file.py new file mode 100644 index 0000000..7380c21 --- /dev/null +++ b/geonode/documents/migrations/0032_remove_document_doc_file.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2 on 2021-05-31 14:17 + +from django.db import migrations +from geonode.base.models import ResourceBaseManager + + +def copy_doc_files(apps, _): + Document = apps.get_model('documents', 'Document') + for _doc in Document.objects.all(): + if _doc.doc_file: + ResourceBaseManager.upload_files( + resource_id=_doc.resourcebase_ptr_id, files=[_doc.doc_file.path, ], force=True) + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0031_auto_20201107_2241'), + ('base', '0062_resourcebase_files'), + ] + + operations = [ + migrations.RunPython(copy_doc_files, migrations.RunPython.noop), + migrations.RemoveField( + model_name='document', + name='doc_file', + ), + ] diff --git a/geonode/documents/migrations/0033_auto_20180414_2120.py b/geonode/documents/migrations/0033_auto_20180414_2120.py new file mode 100644 index 0000000..1d3255e --- /dev/null +++ b/geonode/documents/migrations/0033_auto_20180414_2120.py @@ -0,0 +1,22 @@ +# Generated by Django 1.11.11 on 2018-04-14 21:20\ + + +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0032_auto_20180412_0822'), + ] + + operations = [ + migrations.AlterModelManagers( + name='document', + managers=[ + ('objects', django.db.models.manager.Manager()), + ('base_objects', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/geonode/documents/migrations/0033_remove_document_doc_type.py b/geonode/documents/migrations/0033_remove_document_doc_type.py new file mode 100644 index 0000000..2609d3c --- /dev/null +++ b/geonode/documents/migrations/0033_remove_document_doc_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-07-06 10:47 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0032_remove_document_doc_file'), + ('base', '0067_resourcebase_storetype'), + ] + + operations = [ + migrations.RemoveField( + model_name='document', + name='doc_type', + ), + ] diff --git a/geonode/documents/migrations/0034_auto_20190329_1652.py b/geonode/documents/migrations/0034_auto_20190329_1652.py new file mode 100644 index 0000000..c5140ea --- /dev/null +++ b/geonode/documents/migrations/0034_auto_20190329_1652.py @@ -0,0 +1,23 @@ +# Generated by Django 1.11.20 on 2019-03-29 16:52 + + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0033_auto_20180414_2120'), + ] + + operations = [ + migrations.AlterModelOptions( + name='document', + options={'base_manager_name': 'objects'}, + ), + migrations.AlterModelManagers( + name='document', + managers=[ + ], + ), + ] diff --git a/geonode/documents/migrations/0035_auto_20190404_0820.py b/geonode/documents/migrations/0035_auto_20190404_0820.py new file mode 100644 index 0000000..30995f7 --- /dev/null +++ b/geonode/documents/migrations/0035_auto_20190404_0820.py @@ -0,0 +1,26 @@ +# Generated by Django 1.11.20 on 2019-04-04 08:20 + + +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0034_auto_20190329_1652'), + ] + + operations = [ + migrations.AlterModelOptions( + name='document', + options={}, + ), + migrations.AlterModelManagers( + name='document', + managers=[ + ('objects', django.db.models.manager.Manager()), + ('base_objects', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/geonode/documents/migrations/0036_clean_document_thumbnails.py b/geonode/documents/migrations/0036_clean_document_thumbnails.py new file mode 100644 index 0000000..edfd684 --- /dev/null +++ b/geonode/documents/migrations/0036_clean_document_thumbnails.py @@ -0,0 +1,33 @@ +import logging + +from django.db import migrations + +from ..models import Document + +logger = logging.getLogger(__name__) + + +def set_null_to_non_image_docs_thumbnails(apps, _): + "Sets thumbnail_url to null for documents which are not images" + try: + # update thumbnail urls + for document in Document.objects.all(): + if not document.is_image: + document.thumbnail_url=None + document.save() + # Remove thumbnail links + link_model = apps.get_model('base', 'Link') + link_model.objects.filter(resource__thumbnail_url__isnull=True, name='Thumbnail').delete() + except Exception as e: + logger.exception(e) + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0033_remove_document_doc_type'), + ] + + operations = [ + migrations.RunPython(set_null_to_non_image_docs_thumbnails, migrations.RunPython.noop), + ] diff --git a/geonode/documents/migrations/0037_delete_documentresourcelink.py b/geonode/documents/migrations/0037_delete_documentresourcelink.py new file mode 100644 index 0000000..352c554 --- /dev/null +++ b/geonode/documents/migrations/0037_delete_documentresourcelink.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.21 on 2023-10-05 15:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "0036_clean_document_thumbnails"), + ("base", "0086_linkedresource"), + ] + + operations = [ + migrations.RunSQL( + "INSERT INTO base_linkedresource(source_id, target_id, internal) " + "SELECT document_id, object_id, false as internal " + "FROM documents_documentresourcelink " + "WHERE EXISTS (SELECT 1 FROM base_resourcebase base WHERE base.id=object_id);" + ), + migrations.DeleteModel( + name="DocumentResourceLink", + ), + ] diff --git a/geonode/documents/migrations/0038_alter_document_doc_url.py b/geonode/documents/migrations/0038_alter_document_doc_url.py new file mode 100644 index 0000000..8b29c3b --- /dev/null +++ b/geonode/documents/migrations/0038_alter_document_doc_url.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.22 on 2023-12-05 10:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0037_delete_documentresourcelink'), + ] + + operations = [ + migrations.AlterField( + model_name='document', + name='doc_url', + field=models.URLField(blank=True, help_text='The URL of the document if it is external.', max_length=512, null=True, verbose_name='URL'), + ), + ] diff --git a/geonode/documents/migrations/24_initial.py b/geonode/documents/migrations/24_initial.py new file mode 100644 index 0000000..3993aaa --- /dev/null +++ b/geonode/documents/migrations/24_initial.py @@ -0,0 +1,35 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('base', '__first__'), + ] + + operations = [ + migrations.CreateModel( + name='Document', + fields=[ + ('resourcebase_ptr', models.OneToOneField(parent_link=True, on_delete=models.CASCADE, + auto_created=True, primary_key=True, serialize=False, to='base.ResourceBase')), + ('title_en', models.CharField(help_text='name by which the cited resource is known', max_length=255, null=True, verbose_name='title')), + ('abstract_en', models.TextField(help_text='brief narrative summary of the content of the resource(s)', null=True, verbose_name='abstract', blank=True)), + ('purpose_en', models.TextField(help_text='summary of the intentions with which the resource(s) was developed', null=True, verbose_name='purpose', blank=True)), + ('constraints_other_en', models.TextField(help_text='other restrictions and legal prerequisites for accessing and using the resource or metadata', null=True, verbose_name='restrictions other', blank=True)), + ('supplemental_information_en', models.TextField(default='No information provided', help_text='any other descriptive information about the dataset', null=True, verbose_name='supplemental information')), + ('data_quality_statement_en', models.TextField(help_text="general explanation of the data producer's knowledge about the lineage of a dataset", null=True, verbose_name='data quality statement', blank=True)), + ('object_id', models.PositiveIntegerField(null=True, blank=True)), + ('doc_file', models.FileField(max_length=255, upload_to='documents', null=True, verbose_name='File', blank=True)), + ('extension', models.CharField(max_length=128, null=True, blank=True)), + ('doc_type', models.CharField(max_length=128, null=True, blank=True)), + ('doc_url', models.URLField(help_text='The URL of the document if it is external.', max_length=255, null=True, verbose_name='URL', blank=True)), + ('content_type', models.ForeignKey(blank=True, to='contenttypes.ContentType', on_delete=models.CASCADE, null=True)), + ], + options={ + 'abstract': False, + }, + bases=('base.resourcebase',), + ), + ] diff --git a/geonode/documents/migrations/25_add_documentresourcelink_table.py b/geonode/documents/migrations/25_add_documentresourcelink_table.py new file mode 100644 index 0000000..bc79f3a --- /dev/null +++ b/geonode/documents/migrations/25_add_documentresourcelink_table.py @@ -0,0 +1,25 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('documents', '24_initial'), + ] + + operations = [ + migrations.CreateModel( + name='DocumentResourceLink', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('object_id', models.PositiveIntegerField()), + ('content_type', models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)), + ], + ), + migrations.AddField( + model_name='documentresourcelink', + name='document', + field=models.ForeignKey(related_name='links', to='documents.Document', on_delete=models.CASCADE), + ), + ] diff --git a/geonode/documents/migrations/26_move_data_to_documentresourcelink_table.py b/geonode/documents/migrations/26_move_data_to_documentresourcelink_table.py new file mode 100644 index 0000000..efe50d8 --- /dev/null +++ b/geonode/documents/migrations/26_move_data_to_documentresourcelink_table.py @@ -0,0 +1,29 @@ +from django.db import migrations + + +def forwards(apps, schema_editor): + Document = apps.get_model('documents', 'Document') + DocumentResourceLink = apps.get_model('documents', 'DocumentResourceLink') + + DocumentResourceLink.objects.bulk_create( + DocumentResourceLink( + document_id=document.id, + content_type_id=document.content_type_id, + object_id=document.object_id, + ) + for document in Document.objects.exclude( + content_type_id__isnull=True, + object_id__isnull=True, + ) + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '25_add_documentresourcelink_table'), + ] + + operations = [ + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/geonode/documents/migrations/27_drop_resource_columns_from_document_table.py b/geonode/documents/migrations/27_drop_resource_columns_from_document_table.py new file mode 100644 index 0000000..f59f16b --- /dev/null +++ b/geonode/documents/migrations/27_drop_resource_columns_from_document_table.py @@ -0,0 +1,19 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '26_move_data_to_documentresourcelink_table'), + ] + + operations = [ + migrations.RemoveField( + model_name='document', + name='content_type', + ), + migrations.RemoveField( + model_name='document', + name='object_id', + ), + ] diff --git a/geonode/documents/migrations/__init__.py b/geonode/documents/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/documents/models.py b/geonode/documents/models.py new file mode 100644 index 0000000..565eb32 --- /dev/null +++ b/geonode/documents/models.py @@ -0,0 +1,154 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from urllib.parse import urljoin + +from django.conf import settings +from django.db import models +from django.urls import reverse +from django.utils.functional import classproperty +from django.utils.translation import gettext_lazy as _ + +from geonode.assets.models import LocalAsset +from geonode.client.hooks import hookset +from geonode.base.models import ResourceBase +from geonode.groups.conf import settings as groups_settings +from geonode.documents.enumerations import DOCUMENT_TYPE_MAP, DOCUMENT_MIMETYPE_MAP +from geonode.security.permissions import VIEW_PERMISSIONS, OWNER_PERMISSIONS, DOWNLOAD_PERMISSIONS +from geonode.utils import build_absolute_uri + +logger = logging.getLogger(__name__) + + +class Document(ResourceBase): + """ + A document is any kind of information that can be attached to a map such as pdf, images, videos, xls... + """ + + extension = models.CharField(max_length=128, blank=True, null=True) + + doc_url = models.URLField( + blank=True, + null=True, + max_length=512, + help_text=_("The URL of the document if it is external."), + verbose_name=_("URL"), + ) + + def __str__(self): + return str(self.title) + + def get_absolute_url(self): + return hookset.document_detail_url(self) + + @classproperty + def allowed_permissions(cls): + return { + "anonymous": VIEW_PERMISSIONS + DOWNLOAD_PERMISSIONS, + "default": OWNER_PERMISSIONS + DOWNLOAD_PERMISSIONS, + groups_settings.REGISTERED_MEMBERS_GROUP_NAME: OWNER_PERMISSIONS + DOWNLOAD_PERMISSIONS, + } + + @classproperty + def compact_permission_labels(cls): + return { + "none": _("None"), + "view": _("View Metadata"), + "download": _("View and Download"), + "edit": _("Edit"), + "manage": _("Manage"), + "owner": _("Owner"), + } + + @property + def files(self): + asset = LocalAsset.objects.filter(link__resource=self).first() + return asset.location if asset else [] + + @property + def name(self): + if not self.title and len(self.files) > 0: + return self.files[0].split("/")[-1] + else: + return self.title + + @property + def name_long(self): + if not self.title: + return str(self.id) + else: + return f"{self.title} ({self.id})" + + @property + def href(self): + if self.doc_url: + return self.doc_url + elif self.files: + return urljoin(settings.SITEURL, reverse("document_link", args=(self.id,))) + + @property + def is_local(self): + return False if self.doc_url else True + + @property + def download_is_ajax_safe(self): + return self.is_local + + @property + def is_file(self): + return self.files and self.extension + + @property + def mime_type(self): + if self.is_file and self.extension.lower() in DOCUMENT_MIMETYPE_MAP: + return DOCUMENT_MIMETYPE_MAP[self.extension.lower()] + return None + + @property + def is_audio(self): + AUDIOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "audio"] + return self.extension and self.extension.lower() in AUDIOTYPES + + @property + def is_image(self): + IMGTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "image"] + return self.extension and self.extension.lower() in IMGTYPES + + @property + def is_video(self): + VIDEOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "video"] + return self.extension and self.extension.lower() in VIDEOTYPES + + @property + def class_name(self): + return self.__class__.__name__ + + @property + def embed_url(self): + return reverse("document_embed", args=(self.id,)) + + @property + def download_url(self): + if self.link_set.filter(resource=self.get_self_resource(), link_type="original").exists(): + return self.link_set.filter(resource=self.get_self_resource(), link_type="original").first().url + return build_absolute_uri(reverse("document_download", args=(self.id,))) + + class Meta(ResourceBase.Meta): + pass diff --git a/geonode/documents/static/documents/audio-placeholder.png b/geonode/documents/static/documents/audio-placeholder.png new file mode 100644 index 0000000..ae00733 Binary files /dev/null and b/geonode/documents/static/documents/audio-placeholder.png differ diff --git a/geonode/documents/static/documents/doc-placeholder.png b/geonode/documents/static/documents/doc-placeholder.png new file mode 100644 index 0000000..be6071b Binary files /dev/null and b/geonode/documents/static/documents/doc-placeholder.png differ diff --git a/geonode/documents/static/documents/docx-placeholder.png b/geonode/documents/static/documents/docx-placeholder.png new file mode 100644 index 0000000..9464793 Binary files /dev/null and b/geonode/documents/static/documents/docx-placeholder.png differ diff --git a/geonode/documents/static/documents/generic-placeholder.png b/geonode/documents/static/documents/generic-placeholder.png new file mode 100644 index 0000000..76c61d0 Binary files /dev/null and b/geonode/documents/static/documents/generic-placeholder.png differ diff --git a/geonode/documents/static/documents/gif-placeholder.png b/geonode/documents/static/documents/gif-placeholder.png new file mode 100644 index 0000000..356f339 Binary files /dev/null and b/geonode/documents/static/documents/gif-placeholder.png differ diff --git a/geonode/documents/static/documents/image-placeholder.png b/geonode/documents/static/documents/image-placeholder.png new file mode 100644 index 0000000..a77c6c8 Binary files /dev/null and b/geonode/documents/static/documents/image-placeholder.png differ diff --git a/geonode/documents/static/documents/jpeg-placeholder.png b/geonode/documents/static/documents/jpeg-placeholder.png new file mode 100644 index 0000000..fc2fb54 Binary files /dev/null and b/geonode/documents/static/documents/jpeg-placeholder.png differ diff --git a/geonode/documents/static/documents/jpg-placeholder.png b/geonode/documents/static/documents/jpg-placeholder.png new file mode 100644 index 0000000..2691343 Binary files /dev/null and b/geonode/documents/static/documents/jpg-placeholder.png differ diff --git a/geonode/documents/static/documents/ods-placeholder.png b/geonode/documents/static/documents/ods-placeholder.png new file mode 100644 index 0000000..5c0598a Binary files /dev/null and b/geonode/documents/static/documents/ods-placeholder.png differ diff --git a/geonode/documents/static/documents/odt-placeholder.png b/geonode/documents/static/documents/odt-placeholder.png new file mode 100644 index 0000000..482fedf Binary files /dev/null and b/geonode/documents/static/documents/odt-placeholder.png differ diff --git a/geonode/documents/static/documents/pdf-placeholder.png b/geonode/documents/static/documents/pdf-placeholder.png new file mode 100644 index 0000000..ef5f742 Binary files /dev/null and b/geonode/documents/static/documents/pdf-placeholder.png differ diff --git a/geonode/documents/static/documents/png-placeholder.png b/geonode/documents/static/documents/png-placeholder.png new file mode 100644 index 0000000..3687c88 Binary files /dev/null and b/geonode/documents/static/documents/png-placeholder.png differ diff --git a/geonode/documents/static/documents/ppt-placeholder.png b/geonode/documents/static/documents/ppt-placeholder.png new file mode 100644 index 0000000..d3501c2 Binary files /dev/null and b/geonode/documents/static/documents/ppt-placeholder.png differ diff --git a/geonode/documents/static/documents/rar-placeholder.png b/geonode/documents/static/documents/rar-placeholder.png new file mode 100644 index 0000000..6f3e69e Binary files /dev/null and b/geonode/documents/static/documents/rar-placeholder.png differ diff --git a/geonode/documents/static/documents/tif-placeholder.png b/geonode/documents/static/documents/tif-placeholder.png new file mode 100644 index 0000000..428f3bb Binary files /dev/null and b/geonode/documents/static/documents/tif-placeholder.png differ diff --git a/geonode/documents/static/documents/tiff-placeholder.png b/geonode/documents/static/documents/tiff-placeholder.png new file mode 100644 index 0000000..33c37a5 Binary files /dev/null and b/geonode/documents/static/documents/tiff-placeholder.png differ diff --git a/geonode/documents/static/documents/txt-placeholder.png b/geonode/documents/static/documents/txt-placeholder.png new file mode 100644 index 0000000..86e0fc1 Binary files /dev/null and b/geonode/documents/static/documents/txt-placeholder.png differ diff --git a/geonode/documents/static/documents/video-placeholder.png b/geonode/documents/static/documents/video-placeholder.png new file mode 100644 index 0000000..29e3e8d Binary files /dev/null and b/geonode/documents/static/documents/video-placeholder.png differ diff --git a/geonode/documents/static/documents/xls-placeholder.png b/geonode/documents/static/documents/xls-placeholder.png new file mode 100644 index 0000000..c7031a9 Binary files /dev/null and b/geonode/documents/static/documents/xls-placeholder.png differ diff --git a/geonode/documents/static/documents/xlsx-placeholder.png b/geonode/documents/static/documents/xlsx-placeholder.png new file mode 100644 index 0000000..51f07d0 Binary files /dev/null and b/geonode/documents/static/documents/xlsx-placeholder.png differ diff --git a/geonode/documents/static/documents/xml-placeholder.png b/geonode/documents/static/documents/xml-placeholder.png new file mode 100644 index 0000000..09631af Binary files /dev/null and b/geonode/documents/static/documents/xml-placeholder.png differ diff --git a/geonode/documents/static/documents/zip-placeholder.png b/geonode/documents/static/documents/zip-placeholder.png new file mode 100644 index 0000000..b2f1a9c Binary files /dev/null and b/geonode/documents/static/documents/zip-placeholder.png differ diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py new file mode 100644 index 0000000..316bdaf --- /dev/null +++ b/geonode/documents/tasks.py @@ -0,0 +1,204 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import io + +from PIL import Image +import fitz + +from celery.utils.log import get_task_logger + +from geonode.celery_app import app +from geonode.storage.manager import StorageManager +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset + +from ..base.models import ResourceBase +from .models import Document + +logger = get_task_logger(__name__) + + +class DocumentRenderer: + FILETYPES = ["pdf"] + # See https://pillow.readthedocs.io/en/stable/reference/ImageOps.html#PIL.ImageOps.fit + CROP_CENTERING = {"pdf": (0.0, 0.0)} + + def __init__(self) -> None: + pass + + def supports(self, filename): + return self._get_filetype(filename) in self.FILETYPES + + def render(self, filename): + content = None + if self.supports(filename): + filetype = self._get_filetype(filename) + render = getattr(self, f"render_{filetype}") + content = render(filename) + return content + + def render_pdf(self, filename): + try: + doc = fitz.open(filename) + pix = doc[0].get_pixmap(matrix=fitz.Matrix(2.0, 2.0)) + return pix.pil_tobytes(format="PNG") + except Exception as e: + logger.warning(f"Cound not generate thumbnail for {filename}: {e}") + return None + + def preferred_crop_centering(self, filename): + return self.CROP_CENTERING.get(self._get_filetype(filename)) + + def _get_filetype(self, filname): + return os.path.splitext(filname)[1][1:] + + +doc_renderer = DocumentRenderer() + + +@app.task( + bind=True, + name="geonode.documents.tasks.create_document_thumbnail", + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def create_document_thumbnail(self, object_id): + """ + Create thumbnail for a document. + """ + logger.debug(f"Generating thumbnail for document #{object_id}.") + + default_storage_manager = StorageManager() + + try: + document = Document.objects.get(id=object_id) + except Document.DoesNotExist: + logger.error(f"Document #{object_id} does not exist.") + raise + + image_file = None + thumbnail_content = None + remove_tmp_file = False + centering = (0.5, 0.5) + + doc_path = None + + # get asset of the resource + asset = get_default_asset(document) + if not asset and not document.doc_url: + raise Exception("Document has neither an associated Asset nor a link, cannot generate thumbnail") + + if asset: + handler = asset_handler_registry.get_handler(asset) + asset_storage_manager = handler.get_storage_manager(asset) + doc_path = asset_storage_manager.path(asset.location[0]) + elif document.doc_url: + doc_path = document.doc_url + remove_tmp_file = True + asset_storage_manager = default_storage_manager + + if document.is_image: + try: + image_file = asset_storage_manager.open(doc_path) + except Exception as e: + logger.debug(f"Could not generate thumbnail from remote document {document.doc_url}: {e}") + + if image_file: + try: + image = Image.open(image_file) + with io.BytesIO() as output: + image.save(output, format="PNG") + thumbnail_content = output.getvalue() + output.close() + except Exception as e: + logger.debug(f"Could not generate thumbnail: {e}") + finally: + if image_file is not None: + image_file.close() + if remove_tmp_file: + default_storage_manager.delete(doc_path) + + elif doc_renderer.supports(doc_path): + # in case it's a remote document we want to retrieve it first + if document.doc_url: + doc_path = default_storage_manager.open(doc_path).name + remove_tmp_file = True + try: + thumbnail_content = doc_renderer.render(doc_path) + preferred_centering = doc_renderer.preferred_crop_centering(doc_path) + if preferred_centering is not None: + centering = preferred_centering + except Exception as e: + print(e) + finally: + if remove_tmp_file: + default_storage_manager.delete(doc_path) + if not thumbnail_content: + logger.warning(f"Thumbnail for document #{object_id} empty.") + ResourceBase.objects.filter(id=document.id).update(thumbnail_url=None) + else: + filename = f"document-{document.uuid}-thumb.jpg" + document.save_thumbnail(filename, thumbnail_content, centering=centering) + logger.debug(f"Thumbnail for document #{object_id} created.") + + +@app.task( + bind=True, + name="geonode.documents.tasks.delete_orphaned_document_files", + queue="cleanup", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def delete_orphaned_document_files(self): + from geonode.documents.utils import delete_orphaned_document_files + + delete_orphaned_document_files() + + +@app.task( + bind=True, + name="geonode.documents.tasks.delete_orphaned_thumbnails", + queue="cleanup", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def delete_orphaned_thumbnails(self): + from geonode.base.utils import delete_orphaned_thumbs + + delete_orphaned_thumbs() diff --git a/geonode/documents/templates/documents/document_base.html b/geonode/documents/templates/documents/document_base.html new file mode 100644 index 0000000..cc68e99 --- /dev/null +++ b/geonode/documents/templates/documents/document_base.html @@ -0,0 +1,11 @@ +{% extends "geonode_base.html" %} +{% load i18n %} + +{% block title %} {{ block.super }} {% endblock %} + +{% block body_class %}documents{% endblock %} + +{% block body_outer %} + {% block body %}{% endblock body %} + {% block sidebar %}{% endblock sidebar %} +{% endblock body_outer %} diff --git a/geonode/documents/templates/documents/document_embed.html b/geonode/documents/templates/documents/document_embed.html new file mode 100644 index 0000000..52732d3 --- /dev/null +++ b/geonode/documents/templates/documents/document_embed.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/geonode/documents/templates/documents/document_metadata.html b/geonode/documents/templates/documents/document_metadata.html new file mode 100644 index 0000000..5e2a166 --- /dev/null +++ b/geonode/documents/templates/documents/document_metadata.html @@ -0,0 +1,86 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load bootstrap_tags %} +{% load base_tags %} +{% load guardian_tags %} +{% load floppyforms %} + +{% block title %}{{ document.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock body_class %} + +{% block body_outer %} + +{{ block.super }} + +
    +

    {% trans "Metadata" %} {% blocktrans with document.title as document_title %}for {{ document_title }}{% endblocktrans %}

    +
    + {% trans "Completeness" %} +
    + + +
    +
    +
    + 70% +
    +
    +
    + +
    + +
    + {% if document_form.errors or category_form.errors or metadata_author_form.errors or poc.errors %} +

    {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %}

    +
      + {% if metadata_author_form.errors %} +
    • {% trans "Metadata Author" %}
      • + {{ metadata_author_form.errors }} + {% endif %} + {% if poc_form.errors %} +
    • {% trans "Point of Contact" %}
      • + {{ poc_form.errors }} + {% endif %} + {% for field in document_form %} + {% if field.errors %} +
    • {{ field.label }}
      • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
    • {{ category_form.errors.as_ul }}
      • + {% endif %} +
    + {% endif %} + + {% csrf_token %} +
    + {% form document_form using panel_template %} + {# document_form|as_bootstrap #} +
    + +
    +
    + + + +
    + + + >" %}"/> +
    +
    +
    +
    +
    + +{% endblock body_outer %} diff --git a/geonode/documents/templates/documents/document_metadata_advanced.html b/geonode/documents/templates/documents/document_metadata_advanced.html new file mode 100644 index 0000000..ee6d226 --- /dev/null +++ b/geonode/documents/templates/documents/document_metadata_advanced.html @@ -0,0 +1,154 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load static %} +{% load base_tags %} +{% load bootstrap_tags %} +{% load guardian_tags %} +{% load client_lib_tags %} + +{% block title %}{{ document.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock %} + +{% block body_outer %} + +{{ block.super }} + + + + + + + + + + +
    + {% trans "Explore Documents" %} +

    {% trans "Edit Metadata" %}

    +
    +
    +
    +

    + {% trans "Editing details for" %} {{ document.title }} +

    + +
    + {% if document_form.errors or category_form.errors or metadata_author_form.errors or poc.errors %} +

    {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %}

    +
      + {% if metadata_author_form.errors %} +
    • {% trans "Metadata Author" %}
      • + {{ metadata_author_form.errors }} + {% endif %} + {% if poc_form.errors %} +
    • {% trans "Point of Contact" %}
      • + {{ poc_form.errors }} + {% endif %} + {% for field in document_form %} + {% if field.errors %} +
    • {{ field.label }}
      • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
    • {{ category_form.errors.as_ul }}
      • + {% endif %} +
    + {% endif %} +
    + {% trans "Return to Document" %} + +
    + {% csrf_token %} + +
    + {% block document_fields %} + {% for field in document_form %} + {% if field.name != 'use_featureinfo_custom_template' and field.name != 'featureinfo_custom_template' and field.name not in ADVANCED_EDIT_EXCLUDE_FIELD %} + {% if field.name == 'featured' and not user.is_superuser %} + {% else %} +
    +
    + + {{ field }} +
    +
    + {% endif %} + {% endif %} + {% endfor %} + {% endblock document_fields %} + + + {% block thesauri %} + {# dataset_form|as_bootstrap #} + {% if THESAURI_FILTERS %} + {% for field in tkeywords_form %} +
    +

    + + {{ field }} +

    +
    + {% endfor %} + {% endif %} + + {% endblock thesauri %} +
    +
    +
    + +
    + {% autoescape off %} + {% for choice in category_form.category_choice_field.field.choices %} +
    + +
    + {% endfor %} + {% endautoescape %} +
    +
    + +
    + + +
    + {% trans "Return to Document" %} + +
    +
    +
    +
    +
    +
    +{% endblock %} + +{% block extra_script %} +{{ block.super }} + + +{% endblock extra_script %} diff --git a/geonode/documents/templates/documents/document_metadata_detail.html b/geonode/documents/templates/documents/document_metadata_detail.html new file mode 100644 index 0000000..6e889f8 --- /dev/null +++ b/geonode/documents/templates/documents/document_metadata_detail.html @@ -0,0 +1,16 @@ +{% extends "metadata_detail.html" %} +{% load i18n %} +{% block header %} +{% if 'preview' not in request.GET %} + {% include 'geonode-mapstore-client/snippets/header.html' %} +{% endif %} +{% endblock %} +{% block metaget_absolute_url %} +
    {% trans "Metadata Page" %}
    +
    {% url "document_metadata_detail" resource.id %}
    +{% endblock metaget_absolute_url %} +{% block footer %} +{% if 'preview' not in request.GET %} + {% include 'geonode-mapstore-client/snippets/footer.html' %} +{% endif %} +{% endblock %} diff --git a/geonode/documents/templates/documents/document_replace.html b/geonode/documents/templates/documents/document_replace.html new file mode 100644 index 0000000..9f9a717 --- /dev/null +++ b/geonode/documents/templates/documents/document_replace.html @@ -0,0 +1,32 @@ +{% extends "documents/document_upload_base.html" %} +{% load i18n %} + +{% block title %} {% trans "Replace Document" %} - {{ block.super }} {% endblock %} + +{% block body_class %}documents replace{% endblock body_class %} + +{% block head %} + {{ block.super }} +{% endblock %} + +{% block body %} +
    +
    +

    {% trans "Replace " %}{{ document.title }}

    +
    +
    + + {{ form.errors }} + {% if not document.doc_url %} +

    Currently: {{ document.href}}

    + {{ form.doc_file }} +
    + + +
    + {% endif %} +
    +
    +
    +
    +{% endblock %} \ No newline at end of file diff --git a/geonode/documents/templates/documents/document_thumb_upload.html b/geonode/documents/templates/documents/document_thumb_upload.html new file mode 100644 index 0000000..294e5b8 --- /dev/null +++ b/geonode/documents/templates/documents/document_thumb_upload.html @@ -0,0 +1,94 @@ +{% extends "documents/document_upload_base.html" %} +{% load static %} +{% load bootstrap_tags %} +{% load i18n %} +{% load client_lib_tags %} + +{% block title %} {% trans "Upload Document's Thumbnail" %} - {{ block.super }} {% endblock %} + +{% block body_class %}data documents upload{% endblock body_class %} + +{% block body_outer %} +
    + {% trans "Explore Documents" %} +

    {% trans "Upload Document's Thumbnail" %}

    +
    + +
    + {% block body %} +
    + {% block additional_info %}{% endblock %} + + {% if errors %} +
    + {% for error in errors %} +

    {{ error }}

    + {% endfor %} +
    + {% endif %} + +
    + +
    +


    {% trans "Drop files here" %}

    +
    + +

    {% trans " or select them one by one:" %}

    + +
    + + + + +
    + +
    +
      +

      {% trans "Files to be uploaded" %}

      +
      + +
      + +
      + {% trans "Clear" %} + {% trans "Upload files" %} +
      +
      + + {% endblock %} +
      + +{% endblock body_outer %} + +{% block extra_script %} + {{ block.super }} + + + + +{% endblock extra_script %} diff --git a/geonode/documents/templates/documents/document_upload_base.html b/geonode/documents/templates/documents/document_upload_base.html new file mode 100644 index 0000000..a7f3198 --- /dev/null +++ b/geonode/documents/templates/documents/document_upload_base.html @@ -0,0 +1,27 @@ +{% extends "geonode_base.html" %} +{% load i18n %} +{% load base_tags %} +{% load guardian_tags %} +{% load pagination_tags %} +{% load client_lib_tags %} + +{% block title %} {{ block.super }} {% endblock %} + +{% block body_class %}documents{% endblock %} + +{% block body_outer %} +
      + {% trans "Explore Documents" %} +

      {% trans "Upload Documents" %}

      +

      {% trans "Allowed document types" %}:

      +

      + {% for doc in ALLOWED_DOC_TYPES %} + .{{ doc }} + {% endfor %} +

      +
      +
      + {% block body %}{% endblock body %} + {% block sidebar %}{% endblock sidebar %} +
      +{% endblock body_outer %} diff --git a/geonode/documents/templates/layouts/doc_panels.html b/geonode/documents/templates/layouts/doc_panels.html new file mode 100644 index 0000000..9168477 --- /dev/null +++ b/geonode/documents/templates/layouts/doc_panels.html @@ -0,0 +1,646 @@ +{% load i18n %} +{% load static %} +{% load floppyforms %} +{% load base_tags %} +{% load contact_roles %} + + + + + + + + + + + + + + + + + + +{% block body_outer %} + + +
      +
      +
      + +
      + {% trans "Mandatory" %} +
      +
      + {% trans "Mandatory" %} +
      + {% if UI_REQUIRED_FIELDS %} +
      + {% trans "Mandatory" %} +
      + {% else %} +
      + {% trans "Optional" %} +
      + {% endif %} +
      + + {% block mandatory %} +
      + +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + {% block doc_thumbnail %} +
      + + +
      + {% endblock doc_thumbnail %} +
      + {% block doc_title %} +
      + + + {{ document_form.title }} +
      + {% endblock doc_title %} + {% block doc_abstract %} +
      + + + {{ document_form.abstract }} +
      + {% endblock doc_abstract %} +
      +
      + {% block doc_date_type %} +
      + + + {{ document_form.date_type }} +
      + {% endblock doc_date_type %} + + {% block doc_date %} +
      + + + {{ document_form.date }} +
      + {% endblock doc_date %} + {% block doc_category %} +
      + + +
      + {% endblock doc_category %} + {% block doc_group %} +
      + + +
      + {% endblock doc_group %} + {% block doc_keywords %} +
      + + {{ document_form.keywords }} +
      + {% endblock doc_keywords %} + {% if THESAURI_FILTERS %} +
      + {{ tkeywords_form.as_p }} +
      + {% endif %} +
      +
      +
      +
      +
      +
      +
      +
      + + {% endblock mandatory %} + {% block advanced %} +
      +
      +
      + +
      +
      +
      +
      + {% block doc_language %} +
      + + + {{ document_form.language }} +
      + {% endblock doc_language %} + {% block doc_license %} +
      + + + {{ document_form.license }} +
      + {% endblock doc_license %} + {% block doc_attribution %} +
      + + {{ document_form.attribution }} +
      + {% endblock doc_attribution %} +
      +
      + {% block doc_regions %} +
      + + {{ document_form.regions }} +
      + {% endblock doc_regions %} + {% block doc_data_quality_statement %} +
      + + + {{ document_form.data_quality_statement }} +
      + {% endblock doc_data_quality_statement %} +
      +
      + {% block doc_restriction_code_type %} +
      + + + {{ document_form.restriction_code_type }} +
      + {% endblock doc_restriction_code_type %} + {% block doc_constraints_other %} +
      + + + {{ document_form.constraints_other }} +
      + {% endblock doc_constraints_other %} +
      +
      +
      +
      +
      +
      +
      +
      +
      + + {% endblock advanced %} + {% block ownership %} +
      +
      +
      +
      +

      {% trans "Other, Optional, Metadata" %}

      + {% block doc_edition %} +
      + + + {{ document_form.edition }} +
      + {% endblock doc_edition %} + {% block doc_doi %} +
      + + {{ document_form.doi }} +
      + {% endblock doc_doi %} + {% block doc_purpose %} +
      + + + {{ document_form.purpose }} +
      + {% endblock doc_purpose %} + {% block doc_supplemental_information %} +
      + + + {{ document_form.supplemental_information }} +
      + {% endblock doc_supplemental_information %} +
      +
      + {% block doc_temporal_block %} +
      + {% block doc_temporal_extent_start %} +
      + + + {{ document_form.temporal_extent_start }} +
      + {% endblock doc_temporal_extent_start %} +
      +
      + {% block doc_temporal_extent_end %} +
      + + + {{ document_form.temporal_extent_end }} +
      + {% endblock doc_temporal_extent_end %} +
      + {% endblock %} +
      + {% block doc_maintenance_frequency %} +
      + + + {{ document_form.maintenance_frequency }} +
      + {% endblock doc_maintenance_frequency %} + {% block doc_spatial_representation_type %} +
      + + + {{ document_form.spatial_representation_type }} +
      + {% endblock doc_spatial_representation_type %} + {% block doc_extra_metadata %} +
      + + {{ document_form.extra_metadata }} +
      + {% endblock doc_extra_metadata %} + {% block doc_linked_resources %} +
      + + {{ document_form.linked_resources }} +
      + {% endblock doc_linked_resources %} +
      +
      +
      +
      +
      {% trans "Responsible Parties" %}
      + {% block document_poc %} +
      + + {{ document_form.poc }} +
      + {% endblock document_poc %} +
      +
      +
      {% trans "Responsible and Permissions" %}
      +
      + {% block document_owner %} +
      + + {{ document_form.owner }} +
      + {% endblock document_owner %} +
      +
      + {% trans "toggle more Contact Roles" %} + {% block document_more_contact_roles %} +
      +
      {% trans "more metadata contact roles" %}
      + {% for contact_role in UI_ROLES_IN_TOGGLE_VIEW %} + {% getattribute document_form contact_role as cr %} +
      +
      + + {{ cr}} +
      +
      + {% endfor %} +
      +
      + {% endblock document_more_contact_roles %} + +
      +
      +
      +
      + {% block extra_metadata_content %} + {% endblock extra_metadata_content %} +
      + {% endblock ownership %} +
      +
      + +
      +
      +
      +
      +
      +
      {% trans "Publishing" %}
      +
      +
      + + {{ document_form.metadata_uploaded_preserve }} +
      +
      + + {{ document_form.is_approved }} +
      +
      + + {{ document_form.is_published }} +
      + {% if user.is_superuser %} +
      + + {{ document_form.featured }} +
      + {% endif %} +
      + + {{ document_form.advertised }} +
      +
      +
      +
      +
      +
      +
      +
      +
      +{% endblock %} diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py new file mode 100644 index 0000000..f5bfa0b --- /dev/null +++ b/geonode/documents/tests.py @@ -0,0 +1,849 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +""" +import os +import io +import json +import gisdata + +from PIL import Image +from io import BytesIO + +from unittest.mock import patch +from urllib.parse import urlparse +from pathlib import Path + +from django.urls import reverse +from django.conf import settings +from django.contrib.auth.models import Group +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile +from django.template.defaultfilters import filesizeformat + +from guardian.shortcuts import get_anonymous_user + +from geonode.assets.utils import create_asset_and_link, get_default_asset +from geonode.base.forms import LinkedResourceForm +from geonode.maps.models import Map +from geonode.layers.models import Dataset +from geonode.compat import ensure_string +from geonode.base.models import License, Region, LinkedResource +from geonode.base.enumerations import SOURCE_TYPE_REMOTE +from geonode.documents.apps import DocumentsAppConfig +from geonode.resource.manager import resource_manager +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.tests.utils import NotificationsTestsHelper +from geonode.documents.enumerations import DOCUMENT_TYPE_MAP +from geonode.documents.models import Document + +from geonode.base.populate_test_data import all_public, create_models, create_single_doc, remove_models +from geonode.upload.api.exceptions import FileUploadLimitException + +from .forms import DocumentCreateForm + + +TEST_GIF = os.path.join(os.path.dirname(__file__), "tests/data/img.gif") + + +class DocumentsTest(GeoNodeBaseTestSupport): + type = "document" + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + perm_spec = { + "users": {"admin": ["change_resourcebase", "change_resourcebase_permissions", "view_resourcebase"]}, + "groups": {}, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + create_models("map") + self.project_root = os.path.abspath(os.path.dirname(__file__)) + self.imgfile = io.BytesIO( + b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" + b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + ) + self.anonymous_user = get_anonymous_user() + + def test_document_mimetypes_rendering(self): + ARCHIVETYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "archive"] + AUDIOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "audio"] + IMGTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "image"] + VIDEOTYPES = [_e for _e, _t in DOCUMENT_TYPE_MAP.items() if _t == "video"] + self.assertIsNotNone(ARCHIVETYPES) + self.assertIsNotNone(AUDIOTYPES) + self.assertIsNotNone(IMGTYPES) + self.assertIsNotNone(VIDEOTYPES) + + # Make sure we won't have template rendering issues + self.assertTrue("dwg" in ARCHIVETYPES) + self.assertTrue("dxf" in ARCHIVETYPES) + self.assertTrue("tif" in ARCHIVETYPES) + self.assertTrue("tiff" in ARCHIVETYPES) + self.assertTrue("pbm" in ARCHIVETYPES) + + @patch("geonode.documents.tasks.create_document_thumbnail") + def test_create_document_with_no_rel(self, thumb): + """Tests the creation of a document with no relations""" + thumb.return_value = True + + superuser = get_user_model().objects.get(pk=2) + c = Document.objects.create(owner=superuser, title="theimg") + _, _ = create_asset_and_link(c, superuser, [TEST_GIF]) + c.set_default_permissions() + self.assertEqual(Document.objects.get(pk=c.id).title, "theimg") + + def test_remote_document_is_marked_remote(self): + """Tests creating an external document set its sourcetype to REMOTE.""" + self.client.login(username="admin", password="admin") + form_data = { + "title": "A remote document through form is remote", + "doc_url": "http://www.geonode.org/map.pdf", + } + + response = self.client.post(reverse("document_upload"), data=form_data) + + self.assertEqual(response.status_code, 302) + + d = Document.objects.get(title="A remote document through form is remote") + self.assertEqual(d.sourcetype, SOURCE_TYPE_REMOTE) + + def test_download_is_not_ajax_safe(self): + """Remote document is mark as not safe.""" + self.client.login(username="admin", password="admin") + form_data = { + "title": "A remote document through form is remote", + "doc_url": "https://development.demo.geonode.org/static/mapstore/img/geonode-logo.svg", + } + + response = self.client.post(reverse("document_upload"), data=form_data) + + self.assertEqual(response.status_code, 302) + + d = Document.objects.get(title="A remote document through form is remote") + self.assertFalse(d.download_is_ajax_safe) + + def test_download_is_ajax_safe(self): + """Remote document is mark as not safe.""" + d = create_single_doc("example_doc_name") + self.assertTrue(d.download_is_ajax_safe) + + def test_create_document_url(self): + """Tests creating an external document instead of a file.""" + + superuser = get_user_model().objects.get(pk=2) + c = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + doc_url="http://geonode.org/map.pdf", + owner=superuser, + title="GeoNode Map", + ), + ) + doc = Document.objects.get(pk=c.id) + self.assertEqual(doc.title, "GeoNode Map") + self.assertEqual(doc.extension, "pdf") + + def test_create_document_url_view(self): + """ + Tests creating and updating external documents. + """ + self.client.login(username="admin", password="admin") + form_data = { + "title": "GeoNode Map", + "permissions": '{"users":{"AnonymousUser": ["view_resourcebase"]},"groups":{}}', + "doc_url": "http://www.geonode.org/map.pdf", + } + + response = self.client.post(reverse("document_upload"), data=form_data) + self.assertEqual(response.status_code, 302) + + d = Document.objects.get(title="GeoNode Map") + self.assertEqual(d.doc_url, "http://www.geonode.org/map.pdf") + + def test_uploaded_csv_with_uppercase_extension(self): + """ + The extension of the file should always be lowercase + """ + + self.client.login(username="admin", password="admin") + try: + with open(os.path.join(os.path.dirname(__file__), "tests/data/test.CSV"), "rb") as f: + data = {"title": "CSV with uppercase extension", "doc_file": f, "extension": "CSV"} + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="CSV with uppercase extension") + # verify that the extension is not lowercase + self.assertEqual(d.extension, "csv") + # be sure that also the file extension is not lowercase + asset = get_default_asset(d) + self.assertEqual(Path(asset.location[0]).suffix, ".csv") + finally: + Document.objects.filter(title="CSV with uppercase extension").delete() + + def test_upload_document_form(self): + """ + Tests the Upload form. + """ + form_data = dict() + form = DocumentCreateForm(data=form_data) + self.assertFalse(form.is_valid()) + + # title is required + self.assertTrue("title" in form.errors) + + # since neither a doc_file nor a doc_url are included __all__ should be + # in form.errors. + self.assertTrue("__all__" in form.errors) + + form_data = { + "title": "GeoNode Map", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + "doc_url": "http://www.geonode.org/map.pdf", + } + + form = DocumentCreateForm(data=form_data) + self.assertTrue(form.is_valid()) + + self.assertTrue(isinstance(form.cleaned_data["permissions"], dict)) + + # if permissions are not JSON serializable, the field should be in + # form.errors. + form_data["permissions"] = "non-json string" + self.assertTrue("permissions" in DocumentCreateForm(data=form_data).errors) + + form_data = { + "title": "GeoNode Map", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + } + + file_data = {"doc_file": SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif")} + form = DocumentCreateForm(form_data, file_data) + self.assertTrue(form.is_valid()) + + # The form should raise a validation error when a url and file is + # present. + form_data["doc_url"] = "http://www.geonode.org/map.pdf" + form = DocumentCreateForm(form_data, file_data) + self.assertFalse(form.is_valid()) + self.assertTrue("__all__" in form.errors) + + def test_replace_document(self): + self.client.login(username="admin", password="admin") + + f = SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif") + response = self.client.post( + reverse("document_upload"), + data={ + "title": "File Doc", + "doc_file": f, + "permissions": '{"users":{"AnonymousUser": ["view_resourcebase"]}}', + }, + follow=True, + ) + self.assertEqual(response.status_code, 200) + + # Replace Document + d = Document.objects.get(title="File Doc") + test_image = Image.new("RGBA", size=(50, 50), color=(155, 0, 0)) + f = SimpleUploadedFile("test_image.png", BytesIO(test_image.tobytes()).read(), "image/png") + response = self.client.post(reverse("document_replace", args=(d.id,)), data={"doc_file": f}) + self.assertEqual(response.status_code, 302) + # Remove document + d.delete() + + def test_non_image_documents_thumbnail(self): + self.client.login(username="admin", password="admin") + try: + with open(os.path.join(f"{self.project_root}", "tests/data/text.txt"), "rb") as f: + data = {"title": "Non img File Doc", "doc_file": f, "extension": "txt"} + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="Non img File Doc") + self.assertIsNone(d.thumbnail_url) + finally: + Document.objects.filter(title="Non img File Doc").delete() + + def test_documents_thumbnail(self): + self.client.login(username="admin", password="admin") + try: + # test image doc + with open(os.path.join(f"{self.project_root}", "tests/data/img.gif"), "rb") as f: + data = { + "title": "img File Doc", + "doc_file": f, + "extension": "gif", + } + with self.settings(THUMBNAIL_SIZE={"width": 400, "height": 200}): + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="img File Doc") + self.assertIsNotNone(d.thumbnail_url) + thumb_file = os.path.join( + settings.MEDIA_ROOT, f"thumbs/{os.path.basename(urlparse(d.thumbnail_url).path)}" + ) + file = Image.open(thumb_file) + self.assertEqual(file.size, (400, 200)) + # check thumbnail qualty and extention + self.assertEqual(file.format, "JPEG") + data = { + "title": "Remote img File Doc", + "doc_url": "https://raw.githubusercontent.com/GeoNode/geonode/master/geonode/documents/tests/data/img.gif", + "extension": "gif", + } + with self.settings(THUMBNAIL_SIZE={"width": 400, "height": 200}): + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="Remote img File Doc") + self.assertIsNotNone(d.thumbnail_url) + thumb_file = os.path.join( + settings.MEDIA_ROOT, f"thumbs/{os.path.basename(urlparse(d.thumbnail_url).path)}" + ) + file = Image.open(thumb_file) + self.assertEqual(file.size, (400, 200)) + # check thumbnail qualty and extention + self.assertEqual(file.format, "JPEG") + # test pdf doc + with open(os.path.join(f"{self.project_root}", "tests/data/pdf_doc.pdf"), "rb") as f: + data = { + "title": "Pdf File Doc", + "doc_file": f, + "extension": "pdf", + } + self.client.post(reverse("document_upload"), data=data) + d = Document.objects.get(title="Pdf File Doc") + self.assertIsNotNone(d.thumbnail_url) + thumb_file = os.path.join( + settings.MEDIA_ROOT, f"thumbs/{os.path.basename(urlparse(d.thumbnail_url).path)}" + ) + file = Image.open(thumb_file) + # check thumbnail qualty and extention + self.assertEqual(file.format, "JPEG") + finally: + Document.objects.filter(title="img File Doc").delete() + Document.objects.filter(title="Pdf File Doc").delete() + + def test_upload_document_form_size_limit(self): + form_data = { + "title": "GeoNode Map", + "permissions": '{"anonymous":"document_readonly","authenticated":"resourcebase_readwrite","users":[]}', + } + test_file = SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif") + test_file.size = settings.DEFAULT_MAX_UPLOAD_SIZE * 5 # Set as a large file + + file_data = {"doc_file": test_file} + + with self.assertRaises(FileUploadLimitException): + form = DocumentCreateForm(form_data, file_data) + + self.assertFalse(form.is_valid()) + expected_error = ( + f"File size size exceeds {filesizeformat(settings.DEFAULT_MAX_UPLOAD_SIZE)}. " + f"Please try again with a smaller file." + ) + self.assertEqual(form.errors, {"doc_file": [expected_error]}) + + def test_document_embed(self): + """/documents/1 -> Test accessing the embed view of a document""" + d = Document.objects.all().first() + d.set_default_permissions() + + response = self.client.get(reverse("document_embed", args=(str(d.id),))) + self.assertEqual(response.status_code, 200) + + def test_access_document_upload_form(self): + """Test the form page is returned correctly via GET request /documents/upload""" + + log = self.client.login(username="bobby", password="bob") + self.assertTrue(log) + response = self.client.get(reverse("document_upload")) + self.assertEqual(response.status_code, 405) + + def test_document_isuploaded(self): + """/documents/upload -> Test uploading a document""" + + f = SimpleUploadedFile("test_img_file.gif", self.imgfile.read(), "image/gif") + m = Map.objects.first() + + self.client.login(username="admin", password="admin") + response = self.client.post( + f"{reverse('document_upload')}?no__redirect=true", + data={ + "doc_file": f, + "title": "uploaded_document", + "q": m.id, + "type": "document", + "permissions": '{"users":{"AnonymousUser": ["view_resourcebase"]}}', + }, + ) + self.assertEqual(response.status_code, 200) + + # Permissions Tests + + def test_set_document_permissions(self): + """Verify that the set_document_permissions view is behaving as expected""" + # Get a document to work with + document = Document.objects.first() + + # Set the Permissions + document.set_permissions(self.perm_spec) + + # Test that the Permissions for anonympus user are set correctly + self.assertFalse(self.anonymous_user.has_perm("view_resourcebase", document.get_self_resource())) + + # Test that previous permissions for users other than ones specified in + # the perm_spec (and the document owner) were removed + current_perms = document.get_all_level_info() + self.assertEqual(len(current_perms["users"]), 1) + + # Test that the User permissions specified in the perm_spec were + # applied properly + for username, perm in self.perm_spec["users"].items(): + user = get_user_model().objects.get(username=username) + self.assertTrue(user.has_perm(perm, document.get_self_resource())) + + @patch("geonode.documents.tasks.create_document_thumbnail") + def test_ajax_document_permissions(self, create_thumb): + """Verify that the ajax_document_permissions view is behaving as expected""" + create_thumb.return_value = True + # Setup some document names to work with + superuser = get_user_model().objects.get(pk=2) + document = resource_manager.create( + None, + resource_type=Document, + defaults=dict(files=[TEST_GIF], owner=superuser, title="theimg", is_approved=True), + ) + document_id = document.id + invalid_document_id = 20 + + # Test that an invalid document is handled for properly + response = self.client.post( + reverse("resource_permissions", args=(invalid_document_id,)), + data=json.dumps(self.perm_spec), + content_type="application/json", + ) + self.assertEqual(response.status_code, 401) + + # Test that GET returns permissions + response = self.client.get(reverse("resource_permissions", args=(document_id,))) + assert "permissions" in ensure_string(response.content) + + # Test that a user is required to have + # documents.change_dataset_permissions + + # First test un-authenticated + response = self.client.post( + reverse("resource_permissions", args=(document_id,)), + data=json.dumps(self.perm_spec), + content_type="application/json", + ) + self.assertEqual(response.status_code, 401) + + # Next Test with a user that does NOT have the proper perms + logged_in = self.client.login(username="bobby", password="bob") + self.assertEqual(logged_in, True) + response = self.client.post( + reverse("resource_permissions", args=(document_id,)), + data=json.dumps(self.perm_spec), + content_type="application/json", + ) + self.assertEqual(response.status_code, 401) + + # Login as a user with the proper permission and test the endpoint + logged_in = self.client.login(username="admin", password="admin") + self.assertEqual(logged_in, True) + response = self.client.post( + reverse("resource_permissions", args=(document_id,)), + data=json.dumps(self.perm_spec), + content_type="application/json", + ) + + # Test that the method returns 200 + self.assertEqual(response.status_code, 200) + + def test_batch_edit(self): + Model = Document + view = "document_batch_metadata" + resources = Model.objects.all()[:3] + ids = ",".join(str(element.pk) for element in resources) + # test non-admin access + self.client.login(username="bobby", password="bob") + response = self.client.get(reverse(view)) + self.assertTrue(response.status_code in (401, 403)) + # test group change + group = Group.objects.first() + self.client.login(username="admin", password="admin") + response = self.client.post( + reverse(view), + data={"group": group.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.group, group) + # test owner change + owner = get_user_model().objects.first() + response = self.client.post( + reverse(view), + data={"owner": owner.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.owner, owner) + # test license change + license = License.objects.first() + response = self.client.post( + reverse(view), + data={"license": license.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.license, license) + # test regions change + region = Region.objects.first() + response = self.client.post( + reverse(view), + data={"region": region.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + if resource.regions.all(): + self.assertTrue(region in resource.regions.all()) + # test language change + language = "eng" + response = self.client.post( + reverse(view), + data={"language": language, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.language, language) + # test keywords change + keywords = "some,thing,new" + response = self.client.post( + reverse(view), + data={"keywords": keywords, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + for word in resource.keywords.all(): + self.assertTrue(word.name in keywords.split(",")) + + +class DocumentModerationTestCase(GeoNodeBaseTestSupport): + def setUp(self): + super().setUp() + self.user = "admin" + self.passwd = "admin" + create_models(type=b"document") + create_models(type=b"map") + self.project_root = os.path.abspath(os.path.dirname(__file__)) + self.document_upload_url = f"{(reverse('document_upload'))}?no__redirect=true" + self.u = get_user_model().objects.get(username=self.user) + self.u.email = "test@email.com" + self.u.is_active = True + self.u.save() + + def _get_input_path(self): + base_path = gisdata.GOOD_DATA + return os.path.join(base_path, "vector", "readme.txt") + + def test_document_upload_redirect(self): + with self.settings(ADMIN_MODERATE_UPLOADS=False): + self.client.login(username=self.user, password=self.passwd) + dname = "document title" + with open(os.path.join(f"{self.project_root}", "tests/data/img.gif"), "rb") as f: + data = { + "title": dname, + "doc_file": f, + "resource": "", + "extension": "gif", + "permissions": "{}", + } + resp = self.client.post(self.document_upload_url, data=data) + self.assertEqual(resp.status_code, 200, resp.content) + content = json.loads(resp.content.decode("utf-8")) + self.assertTrue(content["success"]) + self.assertIn("url", content) + + +class DocumentsNotificationsTestCase(NotificationsTestsHelper): + def setUp(self): + self.user = "admin" + self.passwd = "admin" + create_models(type=b"document") + self.anonymous_user = get_anonymous_user() + self.u = get_user_model().objects.get(username=self.user) + self.u.email = "test@email.com" + self.u.is_active = True + self.u.is_superuser = True + self.u.save() + self.setup_notifications_for(DocumentsAppConfig.NOTIFICATIONS, self.u) + self.norman = get_user_model().objects.get(username="norman") + self.norman.email = "norman@email.com" + self.norman.is_active = True + self.norman.save() + self.setup_notifications_for(DocumentsAppConfig.NOTIFICATIONS, self.norman) + + def testDocumentsNotifications(self): + with self.settings( + EMAIL_ENABLE=True, + NOTIFICATION_ENABLED=True, + NOTIFICATIONS_BACKEND="pinax.notifications.backends.email.EmailBackend", + PINAX_NOTIFICATIONS_QUEUE_ALL=False, + ): + self.clear_notifications_queue() + self.client.login(username=self.user, password=self.passwd) + _d = Document.objects.create(title="test notifications", owner=self.norman) + self.assertTrue(self.check_notification_out("document_created", self.u)) + # Ensure "resource.owner" won't be notified for having created its own document + self.assertFalse(self.check_notification_out("document_created", self.norman)) + + self.clear_notifications_queue() + _d.title = "test notifications 2" + _d.save(notify=True) + self.assertTrue(self.check_notification_out("document_updated", self.u)) + + self.clear_notifications_queue() + + +class DocumentResourceLinkTestCase(GeoNodeBaseTestSupport): + def setUp(self): + create_models(b"document") + create_models(b"map") + create_models(b"dataset") + + self.test_file = io.BytesIO( + b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" + b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + ) + + def test_create_document_with_links(self): + """Tests the creation of document links.""" + superuser = get_user_model().objects.get(pk=2) + + d = Document.objects.create(owner=superuser, title="theimg") + _, _ = create_asset_and_link(d, superuser, [TEST_GIF]) + + self.assertEqual(Document.objects.get(pk=d.id).title, "theimg") + + maps = list(Map.objects.all()) + layers = list(Dataset.objects.all()) + resources = maps + layers + + # create document links + + mixin1 = LinkedResourceForm() + mixin1.instance = d + mixin1.cleaned_data = { + "linked_resources": resources, + } + mixin1.save_linked_resources() + + for resource in resources: + _d = LinkedResource.objects.get(source_id=d.id, target_id=resource.id) + self.assertEqual(_d.target_id, resource.id) + + # update document links + + mixin2 = LinkedResourceForm() + mixin2.instance = d + mixin2.cleaned_data = { + "linked_resources": layers, + } + mixin2.save_linked_resources() + + for resource in layers: + _d = LinkedResource.objects.get(source_id=d.id, target_id=resource.id) + self.assertEqual(_d.target_id, resource.id) + + for resource in maps: + with self.assertRaises(LinkedResource.DoesNotExist): + LinkedResource.objects.get(source_id=d.id, target_id=resource.id) + + +class DocumentViewTestCase(GeoNodeBaseTestSupport): + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + def setUp(self): + self.not_admin = get_user_model().objects.create(username="r-lukaku", is_active=True) + self.not_admin.set_password("very-secret") + self.not_admin.save() + self.test_doc = resource_manager.create( + None, + resource_type=Document, + defaults=dict(files=[TEST_GIF], owner=self.not_admin, title="test", is_approved=True), + ) + self.perm_spec = {"users": {"AnonymousUser": []}} + self.doc_link_url = reverse("document_link", args=(self.test_doc.pk,)) + + def test_that_keyword_multiselect_is_disabled_for_non_admin_users(self): + """ + Test that keyword multiselect widget is disabled when the user is not an admin + when FREETEXT_KEYWORDS_READONLY=True + """ + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) + with self.settings(FREETEXT_KEYWORDS_READONLY=True): + response = self.client.get(url) + self.assertFalse(self.not_admin.is_superuser) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.context["form"]["keywords"].field.disabled) + + def test_that_featured_enabling_and_disabling_for_users(self): + # Non Admins + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) + response = self.client.get(url) + self.assertFalse(self.not_admin.is_superuser) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.context["form"]["featured"].field.disabled) + # Admin + self.client.login(username="admin", password="admin") + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertFalse(response.context["form"]["featured"].field.disabled) + + def test_that_keyword_multiselect_is_not_disabled_for_admin_users(self): + """ + Test that only admin users can create/edit keywords + """ + admin = self.not_admin + admin.is_superuser = True + admin.save() + self.client.login(username=admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) + response = self.client.get(url) + self.assertTrue(admin.is_superuser) + self.assertEqual(response.status_code, 200) + self.assertFalse(response.context["form"]["keywords"].field.disabled) + + def test_that_non_admin_user_can_create_write_to_map_without_keyword(self): + """ + Test that non admin users can write to maps without creating/editing keywords + when FREETEXT_KEYWORDS_READONLY=True + """ + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) + with self.settings(FREETEXT_KEYWORDS_READONLY=True): + response = self.client.post( + url, + data={ + "resource-owner": self.not_admin.id, + "resource-title": "doc", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + }, + ) + self.assertFalse(self.not_admin.is_superuser) + self.assertEqual(response.status_code, 200) + self.test_doc.refresh_from_db() + self.assertEqual("doc", self.test_doc.title) + + def test_that_non_admin_user_cannot_create_edit_keyword(self): + """ + Test that non admin users cannot edit/create keywords when FREETEXT_KEYWORDS_READONLY=True + """ + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) + with self.settings(FREETEXT_KEYWORDS_READONLY=True): + response = self.client.post(url, data={"resource-keywords": "wonderful-keyword"}) + self.assertFalse(self.not_admin.is_superuser) + self.assertEqual(response.status_code, 401) + self.assertEqual(response.content, b"Unauthorized: Cannot edit/create Free-text Keywords") + + def test_that_keyword_multiselect_is_enabled_for_non_admin_users_when_freetext_keywords_readonly_istrue(self): + """ + Test that keyword multiselect widget is not disabled when the user is not an admin + and FREETEXT_KEYWORDS_READONLY=False + """ + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) + with self.settings(FREETEXT_KEYWORDS_READONLY=False): + response = self.client.get(url) + self.assertFalse(self.not_admin.is_superuser) + self.assertEqual(response.status_code, 200) + self.assertFalse(response.context["form"]["keywords"].field.disabled) + + def test_that_non_admin_user_can_create_edit_keyword_when_freetext_keywords_readonly_istrue(self): + """ + Test that non admin users can edit/create keywords when FREETEXT_KEYWORDS_READONLY=False + """ + self.client.login(username=self.not_admin.username, password="very-secret") + url = reverse("document_metadata", args=(self.test_doc.pk,)) + with self.settings(FREETEXT_KEYWORDS_READONLY=False): + response = self.client.post( + url, + data={ + "resource-owner": self.not_admin.id, + "resource-title": "doc", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-keywords": "wonderful-keyword", + }, + ) + self.assertFalse(self.not_admin.is_superuser) + self.assertEqual(response.status_code, 200) + self.test_doc.refresh_from_db() + self.assertEqual("doc", self.test_doc.title) + + def test_document_link_with_permissions(self): + self.test_doc.set_permissions(self.perm_spec) + # Get link as Anonymous user + response = self.client.get(self.doc_link_url) + self.assertEqual(response.status_code, 401) + # Access resource with user logged-in + self.client.login(username=self.not_admin.username, password="very-secret") + response = self.client.get(self.doc_link_url) + self.assertEqual(response.status_code, 200) + # test document link with external url + doc = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + doc_url="http://geonode.org/map.pdf", + owner=self.not_admin, + title="GeoNode Map Doc", + ), + ) + self.assertEqual(doc.href, "http://geonode.org/map.pdf") + + # create original link to external + doc.link_set.create(resource=doc.resourcebase_ptr, link_type="original", url="http://google.com/test") + self.assertEqual(doc.download_url, "http://google.com/test") diff --git a/geonode/documents/tests/data/img.gif b/geonode/documents/tests/data/img.gif new file mode 100644 index 0000000..56959b6 Binary files /dev/null and b/geonode/documents/tests/data/img.gif differ diff --git a/geonode/documents/tests/data/pdf_doc.pdf b/geonode/documents/tests/data/pdf_doc.pdf new file mode 100644 index 0000000..0158787 Binary files /dev/null and b/geonode/documents/tests/data/pdf_doc.pdf differ diff --git a/geonode/documents/tests/data/test.CSV b/geonode/documents/tests/data/test.CSV new file mode 100644 index 0000000..8c039d7 --- /dev/null +++ b/geonode/documents/tests/data/test.CSV @@ -0,0 +1,6 @@ +Username; Identifier;First name;Last name +booker12;9012;Rachel;Booker +grey07;2070;Laura;Grey +johnson81;4081;Craig;Johnson +jenkins46;9346;Mary;Jenkins +smith79;5079;Jamie;Smith \ No newline at end of file diff --git a/geonode/documents/tests/data/text.txt b/geonode/documents/tests/data/text.txt new file mode 100644 index 0000000..2a02d41 --- /dev/null +++ b/geonode/documents/tests/data/text.txt @@ -0,0 +1 @@ +TEST diff --git a/geonode/documents/translation.py b/geonode/documents/translation.py new file mode 100644 index 0000000..0fc9d83 --- /dev/null +++ b/geonode/documents/translation.py @@ -0,0 +1,35 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from modeltranslation.translator import translator, TranslationOptions +from geonode.documents.models import Document + + +class DocumentTranslationOptions(TranslationOptions): + fields = ( + "title", + "abstract", + "purpose", + "constraints_other", + "supplemental_information", + "data_quality_statement", + ) + + +translator.register(Document, DocumentTranslationOptions) diff --git a/geonode/documents/urls.py b/geonode/documents/urls.py new file mode 100644 index 0000000..e30ce9d --- /dev/null +++ b/geonode/documents/urls.py @@ -0,0 +1,40 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.urls import include, re_path +from django.contrib.auth.decorators import login_required + +from .views import DocumentUploadView, DocumentUpdateView +from . import views + +js_info_dict = { + "packages": ("geonode.documents",), +} + +urlpatterns = [ # 'geonode.documents.views', + re_path(r"^(?P\d+)/download/?$", views.document_download, name="document_download"), + re_path(r"^(?P\d+)/link/?$", views.document_link, name="document_link"), + re_path(r"^(?P\d+)/replace$", login_required(DocumentUpdateView.as_view()), name="document_replace"), + re_path(r"^(?P\d+)/embed/?$", views.document_embed, name="document_embed"), + re_path(r"^upload/?$", login_required(DocumentUploadView.as_view()), name="document_upload"), + re_path(r"^(?P[^/]*)/metadata_detail$", views.document_metadata_detail, name="document_metadata_detail"), + re_path(r"^(?P\d+)/metadata$", views.document_metadata, name="document_metadata"), + re_path(r"^metadata/batch/$", views.document_batch_metadata, name="document_batch_metadata"), + re_path(r"^(?P\d+)/metadata_advanced$", views.document_metadata_advanced, name="document_metadata_advanced"), + re_path(r"^", include("geonode.documents.api.urls")), +] diff --git a/geonode/documents/utils.py b/geonode/documents/utils.py new file mode 100644 index 0000000..8f5ec4a --- /dev/null +++ b/geonode/documents/utils.py @@ -0,0 +1,85 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""Utilities for managing GeoNode documents +""" + +# Standard Modules +import os +import logging + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset +from geonode.storage.manager import storage_manager + +# Django functionality +from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from django.template import loader +from django.utils.translation import gettext_lazy as _ +from django.utils.text import slugify + +# Geonode functionality +from geonode.documents.models import Document +from geonode.base import register_event +from geonode.monitoring.models import EventType + +logger = logging.getLogger(__name__) + + +def delete_orphaned_document_files(): + """ + Deletes orphaned files of deleted documents. + """ + deleted = [] + _, files = storage_manager.listdir(os.path.join("documents", "document")) + + for filename in files: + if Document.objects.filter(doc_file__contains=filename).count() == 0: + logger.debug(f"Deleting orphaned document {filename}") + try: + storage_manager.delete(os.path.join(os.path.join("documents", "document"), filename)) + deleted.append(filename) + except NotImplementedError as e: + logger.error(f"Failed to delete orphaned document '{filename}': {e}") + + return deleted + + +def get_download_response(request, docid, attachment=False): + """ + Returns a download response if user has access to download the document of a given id, + and an http response if they have no permissions to download it. + """ + document = get_object_or_404(Document, pk=docid) + + if not request.user.has_perm("base.download_resourcebase", obj=document.get_self_resource()): + return HttpResponse( + loader.render_to_string( + "401.html", context={"error_message": _("You are not allowed to view this document.")}, request=request + ), + status=401, + ) + if attachment: + register_event(request, EventType.EVENT_DOWNLOAD, document) + filename = slugify(os.path.splitext(os.path.basename(document.title))[0]) + + asset = get_default_asset(document) + asset_handler = asset_handler_registry.get_handler(asset) + return asset_handler.get_download_handler(asset).create_response(asset, attachment, basename=filename) diff --git a/geonode/documents/views.py b/geonode/documents/views.py new file mode 100644 index 0000000..5389b6b --- /dev/null +++ b/geonode/documents/views.py @@ -0,0 +1,554 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import json +import shutil +import logging +import warnings +import traceback + +from django.urls import reverse +from django.conf import settings +from django.contrib import messages +from django.shortcuts import render, get_object_or_404 +from django.utils.translation import gettext_lazy as _ +from django.contrib.auth.decorators import login_required +from django.template import loader +from django.views.generic.edit import CreateView, UpdateView +from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.core.exceptions import PermissionDenied, ObjectDoesNotExist + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset +from geonode.base.api.exceptions import geonode_exception_handler +from geonode.client.hooks import hookset +from geonode.utils import mkdtemp, resolve_object +from geonode.base.views import batch_modify +from geonode.people.forms import ProfileForm +from geonode.base import register_event +from geonode.base.bbox_utils import BBOXHelper +from geonode.groups.models import GroupProfile +from geonode.monitoring.models import EventType +from geonode.storage.manager import storage_manager +from geonode.resource.manager import resource_manager +from geonode.decorators import check_keyword_write_perms +from geonode.security.utils import get_user_visible_groups +from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm +from geonode.base.models import Thesaurus, TopicCategory +from geonode.base import enumerations + +from pathlib import Path + +from .utils import get_download_response + +from .models import Document +from .forms import DocumentForm, DocumentCreateForm, DocumentReplaceForm + +logger = logging.getLogger("geonode.documents.views") + +ALLOWED_DOC_TYPES = settings.ALLOWED_DOCUMENT_TYPES + +_PERMISSION_MSG_DELETE = _("You are not permitted to delete this document") +_PERMISSION_MSG_GENERIC = _("You do not have permissions for this document.") +_PERMISSION_MSG_MODIFY = _("You are not permitted to modify this document") +_PERMISSION_MSG_METADATA = _("You are not permitted to modify this document's metadata") +_PERMISSION_MSG_VIEW = _("You are not permitted to view this document") + + +def _resolve_document(request, docid, permission="base.change_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs): + """ + Resolve the document by the provided primary key and check the optional permission. + """ + return resolve_object(request, Document, {"pk": docid}, permission=permission, permission_msg=msg, **kwargs) + + +def document_download(request, docid): + response = get_download_response(request, docid, attachment=True) + return response + + +def document_link(request, docid): + response = get_download_response(request, docid) + return response + + +def document_embed(request, docid): + document = get_object_or_404(Document, pk=docid) + + if not request.user.has_perm("base.download_resourcebase", obj=document.get_self_resource()): + return HttpResponse( + loader.render_to_string( + "401.html", context={"error_message": _("You are not allowed to view this document.")}, request=request + ), + status=401, + ) + if document.is_image: + if document.doc_url: + imageurl = document.doc_url + else: + imageurl = reverse("document_link", args=(document.id,)) + context_dict = { + "image_url": imageurl, + "resource": document.get_self_resource(), + } + return render(request, "documents/document_embed.html", context_dict) + else: + context_dict = { + "document_link": reverse("document_link", args=(document.id,)), + "resource": document.get_self_resource(), + } + return render(request, "documents/document_embed.html", context_dict) + + +class DocumentUploadView(CreateView): + http_method_names = ["post"] + form_class = DocumentCreateForm + + def post(self, request, *args, **kwargs): + self.object = None + try: + return super().post(request, *args, **kwargs) + except Exception as e: + exception_response = geonode_exception_handler(e, {}) + return HttpResponse( + json.dumps(exception_response.data), + content_type="application/json", + status=exception_response.status_code, + ) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["ALLOWED_DOC_TYPES"] = ALLOWED_DOC_TYPES + return context + + def form_invalid(self, form): + messages.error(self.request, f"{form.errors}") + if self.request.GET.get("no__redirect", False): + plaintext_errors = [] + for field in form.errors.values(): + plaintext_errors.append(field.data[0].message) + out = {"success": False} + out["message"] = ".".join(plaintext_errors) + status_code = 400 + return HttpResponse(json.dumps(out), content_type="application/json", status=status_code) + else: + form.name = None + form.title = None + form.doc_file = None + form.doc_url = None + return self.render_to_response(self.get_context_data(request=self.request, form=form)) + + def form_valid(self, form): + """ + If the form is valid, save the associated model. + """ + doc_form = form.cleaned_data + + file = doc_form.pop("doc_file", None) + if file: + tempdir = mkdtemp() + dirname = os.path.basename(tempdir) + name = Path(file.name) + filepath = storage_manager.save(f"{dirname}/{name.stem}{name.suffix.lower()}", file) + storage_path = storage_manager.path(filepath) + self.object = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + owner=self.request.user, + doc_url=doc_form.pop("doc_url", None), + title=doc_form.pop("title", file.name), + description=doc_form.pop("abstract", None), + extension=doc_form.pop("extension", None), + link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES + data_title=doc_form.pop("title", file.name), + data_type=doc_form.pop("extension", None), + files=[storage_path], + ), + ) + + # Removing the temp file + # TODO: creating a file and then cloning it as an Asset may be slow: we may want to + # create the file directly in the asset dir or to move it + logger.info(f"Removing document temp dir {tempdir}") + shutil.rmtree(tempdir, ignore_errors=True) + + else: + self.object = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + owner=self.request.user, + doc_url=doc_form.pop("doc_url", None), + title=doc_form.pop("title", None), + extension=doc_form.pop("extension", None), + sourcetype=enumerations.SOURCE_TYPE_REMOTE, + ), + ) + + self.object.handle_moderated_uploads() + resource_manager.set_permissions( + None, instance=self.object, permissions=form.cleaned_data["permissions"], created=True + ) + + abstract = None + date = None + regions = [] + keywords = [] + bbox = None + url = hookset.document_detail_url(self.object) + + out = {"success": False} + + if getattr(settings, "EXIF_ENABLED", False): + try: + from geonode.documents.exif.utils import exif_extract_metadata_doc + + exif_metadata = exif_extract_metadata_doc(self.object) + if exif_metadata: + date = exif_metadata.get("date", None) + keywords.extend(exif_metadata.get("keywords", [])) + bbox = exif_metadata.get("bbox", None) + abstract = exif_metadata.get("abstract", None) + except Exception: + logger.debug("Exif extraction failed.") + + resource_manager.update( + self.object.uuid, + instance=self.object, + keywords=keywords, + regions=regions, + vals=dict( + abstract=abstract, + date=date, + date_type="Creation", + bbox_polygon=BBOXHelper.from_xy(bbox).as_polygon() if bbox else None, + ), + notify=True, + ) + resource_manager.set_thumbnail(self.object.uuid, instance=self.object, overwrite=False) + + register_event(self.request, EventType.EVENT_UPLOAD, self.object) + + if self.request.GET.get("no__redirect", False): + out["success"] = True + out["url"] = url + if out["success"]: + status_code = 200 + else: + status_code = 400 + return HttpResponse(json.dumps(out), content_type="application/json", status=status_code) + else: + return HttpResponseRedirect(url) + + +class DocumentUpdateView(UpdateView): + template_name = "documents/document_replace.html" + pk_url_kwarg = "docid" + form_class = DocumentReplaceForm + queryset = Document.objects.all() + context_object_name = "document" + + def post(self, request, *args, **kwargs): + self.object = self.get_object() + try: + return super().post(request, *args, **kwargs) + except Exception as e: + exception_response = geonode_exception_handler(e, {}) + return HttpResponse( + json.dumps(exception_response.data), + content_type="application/json", + status=exception_response.status_code, + ) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["ALLOWED_DOC_TYPES"] = ALLOWED_DOC_TYPES + return context + + def form_valid(self, form): + """ + If the form is valid, save the associated model. + """ + doc_form = form.cleaned_data + + file = doc_form.pop("doc_file", None) + if file: + tempdir = mkdtemp() + dirname = os.path.basename(tempdir) + filepath = storage_manager.save(os.path.join(dirname, file.name), file) + storage_path = storage_manager.path(filepath) + self.object = resource_manager.update( + self.object.uuid, instance=self.object, vals=dict(owner=self.request.user) + ) + + # replace data in existing asset + asset = get_default_asset(self.object, link_type="uploaded") + if asset: + asset_handler_registry.get_handler(asset).replace_data(asset, [storage_path]) + + if tempdir != os.path.dirname(storage_path): + shutil.rmtree(tempdir, ignore_errors=True) + + register_event(self.request, EventType.EVENT_CHANGE, self.object) + url = hookset.document_detail_url(self.object) + return HttpResponseRedirect(url) + + +@login_required +@check_keyword_write_perms +def document_metadata( + request, + docid, + template="documents/document_metadata.html", + panel_template="layouts/doc_panels.html", + custom_metadata=None, + ajax=True, +): + document = None + try: + document = _resolve_document(request, docid, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not document: + raise Http404(_("Not found")) + + # Add metadata_author or poc if missing + document.add_missing_metadata_author_or_poc() + + topic_category = document.category + current_keywords = [keyword.name for keyword in document.keywords.all()] + + if request.method == "POST": + document_form = DocumentForm(request.POST, instance=document, prefix="resource", user=request.user) + category_form = CategoryForm( + request.POST, + prefix="category_choice_field", + initial=( + int(request.POST["category_choice_field"]) + if "category_choice_field" in request.POST and request.POST["category_choice_field"] + else None + ), + ) + + if hasattr(settings, "THESAURUS"): + tkeywords_form = TKeywordForm(request.POST) + else: + tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords") + + else: + document_form = DocumentForm(instance=document, prefix="resource", user=request.user) + document_form.disable_keywords_widget_for_non_superuser(request.user) + category_form = CategoryForm( + prefix="category_choice_field", initial=topic_category.id if topic_category else None + ) + + # Keywords from THESAURUS management + doc_tkeywords = document.tkeywords.all() + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + warnings.warn( + "The settings for Thesaurus has been moved to Model, \ + this feature will be removed in next releases", + DeprecationWarning, + ) + tkeywords_list = "" + lang = "en" # TODO: use user's language + if doc_tkeywords and len(doc_tkeywords) > 0: + tkeywords_ids = doc_tkeywords.values_list("id", flat=True) + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + el = settings.THESAURUS + thesaurus_name = el["name"] + try: + t = Thesaurus.objects.get(identifier=thesaurus_name) + for tk in t.thesaurus.filter(pk__in=tkeywords_ids): + tkl = tk.keyword.filter(lang=lang) + if len(tkl) > 0: + tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True))) + tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids + except Exception: + tb = traceback.format_exc() + logger.error(tb) + + tkeywords_form = TKeywordForm(instance=document) + else: + tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords") + # set initial values for thesaurus form + for tid in tkeywords_form.fields: + values = [] + values = [keyword.id for keyword in doc_tkeywords if int(tid) == keyword.thesaurus.id] + tkeywords_form.fields[tid].initial = values + + if request.method == "POST" and document_form.is_valid() and category_form.is_valid() and tkeywords_form.is_valid(): + new_keywords = current_keywords if request.keyword_readonly else document_form.cleaned_data["keywords"] + new_regions = document_form.cleaned_data["regions"] + + new_category = None + if ( + category_form + and "category_choice_field" in category_form.cleaned_data + and category_form.cleaned_data["category_choice_field"] + ): + new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"])) + + # update contact roles + document.set_contact_roles_from_metadata_edit(document_form) + document.save() + + document = document_form.instance + resource_manager.update( + document.uuid, + instance=document, + keywords=new_keywords, + regions=new_regions, + vals=dict(category=new_category), + notify=True, + extra_metadata=json.loads(document_form.cleaned_data["extra_metadata"]), + ) + + resource_manager.set_thumbnail(document.uuid, instance=document, overwrite=False) + document_form.save_linked_resources() + + register_event(request, EventType.EVENT_CHANGE_METADATA, document) + url = hookset.document_detail_url(document) + if not ajax: + return HttpResponseRedirect(url) + message = document.id + + try: + # Keywords from THESAURUS management + # Rewritten to work with updated autocomplete + if not tkeywords_form.is_valid(): + return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400)) + + thesaurus_setting = getattr(settings, "THESAURUS", None) + if thesaurus_setting: + tkeywords_data = tkeywords_form.cleaned_data["tkeywords"] + tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"]) + document.tkeywords.set(tkeywords_data) + elif Thesaurus.objects.all().exists(): + fields = tkeywords_form.cleaned_data + document.tkeywords.set(tkeywords_form.cleanx(fields)) + + except Exception: + tb = traceback.format_exc() + logger.error(tb) + + vals = {} + if "group" in document_form.changed_data: + vals["group"] = document_form.cleaned_data.get("group") + if any([x in document_form.changed_data for x in ["is_approved", "is_published"]]): + vals["is_approved"] = document_form.cleaned_data.get("is_approved", document.is_approved) + vals["is_published"] = document_form.cleaned_data.get("is_published", document.is_published) + resource_manager.update( + document.uuid, + instance=document, + notify=True, + vals=vals, + extra_metadata=json.loads(document_form.cleaned_data["extra_metadata"]), + ) + return HttpResponse(json.dumps({"message": message})) + elif request.method == "POST" and ( + not document_form.is_valid() or not category_form.is_valid() or not tkeywords_form.is_valid() + ): + errors_list = { + **document_form.errors.as_data(), + **category_form.errors.as_data(), + **tkeywords_form.errors.as_data(), + } + logger.error(f"GeoApp Metadata form is not valid: {errors_list}") + out = {"success": False, "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()]} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + # - POST Request Ends here - + + # Request.GET + # define contact role forms + contact_role_forms_context = {} + for role in document.get_multivalue_role_property_names(): + document_form.fields[role].initial = [p.username for p in document.__getattribute__(role)] + role_form = ProfileForm(prefix=role) + role_form.hidden = True + contact_role_forms_context[f"{role}_form"] = role_form + + metadata_author_groups = get_user_visible_groups(request.user) + if not request.user.can_publish(document): + document_form.fields["is_published"].widget.attrs.update({"disabled": "true"}) + if not request.user.can_approve(document): + document_form.fields["is_approved"].widget.attrs.update({"disabled": "true"}) + + register_event(request, EventType.EVENT_VIEW_METADATA, document) + return render( + request, + template, + context={ + "resource": document, + "document": document, + "panel_template": panel_template, + "custom_metadata": custom_metadata, + "document_form": document_form, + "category_form": category_form, + "tkeywords_form": tkeywords_form, + "metadata_author_groups": metadata_author_groups, + "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False), + "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + "UI_MANDATORY_FIELDS": list( + set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", [])) + | set(getattr(settings, "UI_REQUIRED_FIELDS", [])) + ), + **contact_role_forms_context, + "UI_ROLES_IN_TOGGLE_VIEW": document.get_ui_toggled_role_property_names(), + }, + ) + + +@login_required +def document_metadata_advanced(request, docid): + return document_metadata(request, docid, template="documents/document_metadata_advanced.html") + + +def document_metadata_detail(request, docid, template="documents/document_metadata_detail.html", custom_metadata=None): + try: + document = _resolve_document(request, docid, "view_resourcebase", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not document: + raise Http404(_("Not found")) + + group = None + if document.group: + try: + group = GroupProfile.objects.get(slug=document.group.name) + except ObjectDoesNotExist: + group = None + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + register_event(request, EventType.EVENT_VIEW_METADATA, document) + + return render( + request, + template, + context={"resource": document, "group": group, "SITEURL": site_url, "custom_metadata": custom_metadata}, + ) + + +@login_required +def document_batch_metadata(request): + return batch_modify(request, "Document") diff --git a/geonode/facets/__init__.py b/geonode/facets/__init__.py new file mode 100644 index 0000000..da86ef5 --- /dev/null +++ b/geonode/facets/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/facets/apps.py b/geonode/facets/apps.py new file mode 100644 index 0000000..3e07345 --- /dev/null +++ b/geonode/facets/apps.py @@ -0,0 +1,32 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.apps import AppConfig + + +logger = logging.getLogger(__name__) + + +class GeoNodeFacetsConfig(AppConfig): + name = "geonode.facets" + verbose_name = "GeoNode Facets endpoints" + + def ready(self): + super(GeoNodeFacetsConfig, self).ready() diff --git a/geonode/facets/models.py b/geonode/facets/models.py new file mode 100644 index 0000000..a2ae79e --- /dev/null +++ b/geonode/facets/models.py @@ -0,0 +1,162 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.conf import settings + +DEFAULT_FACET_PAGE_SIZE = 10 + +# Well known types of facet - not an enum bc it needs to be extensible +FACET_TYPE_PLACE = "place" +FACET_TYPE_USER = "user" +FACET_TYPE_THESAURUS = "thesaurus" +FACET_TYPE_CATEGORY = "category" +FACET_TYPE_BASE = "base" +FACET_TYPE_KEYWORD = "keyword" +FACET_TYPE_GROUP = "group" + +logger = logging.getLogger(__name__) + + +class FacetProvider: + """ + Provides access to the facet information and the related topics + """ + + def __init__(self, **kwargs): + self.config = kwargs.get("config", {}).copy() + + def __str__(self): + return f"{self.__class__.__name__}[{self.name}]" + + @property + def name(self) -> str: + """ + Get the name of the facet, to be used as a key for this provider. + You may want to override this method in order to have an optimized logic + :return: The name of the provider as a str + """ + self.get_info()["name"] + + def get_info(self, lang="en", **kwargs) -> dict: + """ + Get the basic info for this provider, as a dict with these keys: + - 'name': the name of the provider (the one returned by name()) + - 'filter': the filtering key to be used in a filter query + - 'label': a generic label for the facet; the client should try and localize it whenever possible + - 'localized_label': a localized label for the facet (localized according to the `lang` param) + - 'type': the facet type (e.g. user, region, thesaurus, ...) + - 'hierarchical': boolean value telling if the facet items are hierarchically organized + - "order": an optional integer suggesting the relative ordering of the facets + + :param lang: lanuage for label localization + :return: a dict + """ + pass + + def get_facet_items( + self, + queryset, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + """ + Return the items of the facets, in a tuple: + - int, total number of items matched + - list, topic records. A topic record is a dict having these keys: + - key: the key of the items that should be used for filtering + - label: a generic label for the item; the client should try and localize it whenever possible + - localized_label: a localized label for the item + - count: the count of such topic in the current facet + - other facet specific keys + :param queryset: the prefiltered queryset (may be filtered for authorization or other filters) + :param start: int: pagination, the index of the initial returned item + :param end: int: pagination, the index of the last returned item + :param lang: the preferred language for the labels + :param topic_contains: only returns matching topics + :param keys: only returns topics with given keys, even if their count is 0 + :return: a tuple int:total count of record, list of items + """ + pass + + def get_topics(self, keys: list, lang="en", **kwargs) -> list: + """ + Return the topics with the requested ids as a list + - list, topic records. A topic record is a dict having these keys: + - key: the key of the items that should be used for filtering + - label: a generic label for the item; the client should try and localize it whenever possible + - localized_label: a localized label for the item + - other facet specific keys + :param keys: the list of the keys of the topics, as returned by the get_facet_items() method + :param lang: the preferred language for the labels + :return: list of items + """ + pass + + @classmethod + def register(cls, registry, **kwargs) -> None: + """ + Perform registration of instances of this Provider + :param registry: the registry where instances shall be registered + :param kwargs: other args that may be needed by Providers + """ + pass + + +class FacetsRegistry: + def __init__(self): + self.facet_providers = None + + def _load_facets_configuration(self) -> None: + """ + Facet loading is done lazily because some FacetProvider may need to access the DB, which may not have been + initialized/created yet + """ + from django.utils.module_loading import import_string + + self.facet_providers = dict() + + logger.info("Initializing Facets") + + for providerconf in getattr(settings, "FACET_PROVIDERS", []): + clz = providerconf["class"] + provider = import_string(clz) + provider.register(self, config=providerconf.get("config", {})) + + def register_facet_provider(self, provider: FacetProvider): + logger.info(f"Registering {provider}") + self.facet_providers[provider.get_info()["name"]] = provider + + def get_providers(self): + if self.facet_providers is None: + self._load_facets_configuration() + return self.facet_providers.values() + + def get_provider(self, name): + if self.facet_providers is None: + self._load_facets_configuration() + return self.facet_providers.get(name, None) + + +facet_registry = FacetsRegistry() diff --git a/geonode/facets/providers/__init__.py b/geonode/facets/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/facets/providers/baseinfo.py b/geonode/facets/providers/baseinfo.py new file mode 100644 index 0000000..11df278 --- /dev/null +++ b/geonode/facets/providers/baseinfo.py @@ -0,0 +1,150 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.db.models import Count + +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, FACET_TYPE_BASE + +logger = logging.getLogger(__name__) + + +class ResourceTypeFacetProvider(FacetProvider): + """ + Implements faceting for resources' type and subtype + """ + + @property + def name(self) -> str: + return "resourcetype" + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": self.name, + "filter": "filter{resource_type.in}", + "label": "Resource type", + "type": FACET_TYPE_BASE, + "hierarchical": True, + } + + def get_facet_items( + self, + queryset=None, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for %s", self.name) + + if topic_contains: + logger.warning(f"Facet {self.name} does not support topic_contains filtering") + + q = queryset.values("resource_type", "subtype") + q = q.annotate(ctype=Count("resource_type"), csub=Count("subtype")) + q = q.order_by() + + # aggregate subtypes into rtypes + tree = {} + for r in q.all(): + res_type = r["resource_type"] + t = tree.get(res_type, {"cnt": 0, "sub": {}}) + t["cnt"] += r["ctype"] + if sub := r["subtype"]: + t["sub"][sub] = {"cnt": r["ctype"]} + tree[res_type] = t + + logger.info("Found %d main facets for %s", len(tree), self.name) + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [] + for rtype, info in tree.items(): + t = {"key": rtype, "label": rtype, "count": info["cnt"]} + if sub := info["sub"]: + children = [] + for stype, sinfo in sub.items(): + children.append({"key": stype, "label": stype, "count": sinfo["cnt"]}) + t["filter"] = "filter{subtype.in}" + t["items"] = sorted(children, reverse=True, key=lambda x: x["count"]) + topics.append(t) + + return len(topics), sorted(topics, reverse=True, key=lambda x: x["count"]) + + @classmethod + def register(cls, registry, **kwargs) -> None: + registry.register_facet_provider(ResourceTypeFacetProvider(**kwargs)) + + +class FeaturedFacetProvider(FacetProvider): + """ + Implements faceting for resources flagged as featured + """ + + @property + def name(self) -> str: + return "featured" + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": self.name, + "filter": "filter{featured}", + "label": "Featured", + "type": FACET_TYPE_BASE, + "hierarchical": False, + "order": 0, + } + + def get_facet_items( + self, + queryset=None, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for %s", self.name) + + if topic_contains: + logger.warning(f"Facet {self.name} does not support topic_contains filtering") + + q = queryset.values("featured").annotate(cnt=Count("featured")).order_by() + + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [ + { + "key": r["featured"], + "label": str(r["featured"]), + "count": r["cnt"], + } + for r in q[start:end] + ] + + return 2, topics + + @classmethod + def register(cls, registry, **kwargs) -> None: + registry.register_facet_provider(FeaturedFacetProvider(**kwargs)) diff --git a/geonode/facets/providers/category.py b/geonode/facets/providers/category.py new file mode 100644 index 0000000..56bbea4 --- /dev/null +++ b/geonode/facets/providers/category.py @@ -0,0 +1,115 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.db.models import Count +from django.utils.translation import gettext_lazy as _ + +from geonode.base.models import TopicCategory +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, FACET_TYPE_CATEGORY + +logger = logging.getLogger(__name__) + + +class CategoryFacetProvider(FacetProvider): + """ + Implements faceting for resource's topicCategory + """ + + @property + def name(self) -> str: + return "category" + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": self.name, + "filter": "filter{category.identifier.in}", + "label": _("Category"), + "type": FACET_TYPE_CATEGORY, + } + + def get_facet_items( + self, + queryset, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for %s", self.name) + + filters = {"resourcebase__in": queryset} + + if topic_contains: + filters["gn_description__icontains"] = topic_contains + + if keys: + logger.debug("Filtering by keys %r", keys) + filters["identifier__in"] = keys + + q = ( + TopicCategory.objects.values("identifier", "gn_description", "fa_class") + .filter(**filters) + .annotate(count=Count("resourcebase")) + .order_by("-count") + ) + + logger.debug(" PREFILTERED QUERY ---> %s\n\n", queryset.query) + logger.debug(" ADDITIONAL FILTERS ---> %s\n\n", filters) + logger.debug(" FINAL QUERY ---> %s\n\n", q.query) + + cnt = q.count() + + logger.info("Found %d facets for %s", cnt, self.name) + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [ + { + "key": r["identifier"], + "label": r["gn_description"], + "count": r["count"], + "fa_class": r["fa_class"], + } + for r in q[start:end].all() + ] + + return cnt, topics + + def get_topics(self, keys: list, lang="en", **kwargs) -> list: + q = TopicCategory.objects.filter(identifier__in=keys) + + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + return [ + { + "key": r.identifier, + "label": r.gn_description, + "fa_class": r.fa_class, + } + for r in q.all() + ] + + @classmethod + def register(cls, registry, **kwargs) -> None: + registry.register_facet_provider(CategoryFacetProvider(**kwargs)) diff --git a/geonode/facets/providers/group.py b/geonode/facets/providers/group.py new file mode 100644 index 0000000..fe9592a --- /dev/null +++ b/geonode/facets/providers/group.py @@ -0,0 +1,111 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + + +import logging + +from django.utils.translation import gettext_lazy as _ +from django.db.models import Count + +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, FACET_TYPE_GROUP +from geonode.groups.models import GroupProfile +from geonode.security.utils import get_user_visible_groups + +logger = logging.getLogger(__name__) + + +class GroupFacetProvider(FacetProvider): + """ + Implements faceting for resource's group + """ + + @property + def name(self) -> str: + return "group" + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": self.name, + "filter": "filter{group.in}", + "label": _("Group"), + "type": FACET_TYPE_GROUP, + } + + def get_facet_items( + self, + queryset, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for %s", self.name) + + filters = dict() + if keys: + logger.debug("Filtering by keys %r", keys) + filters["group__id__in"] = keys + + if topic_contains: + filters["group__name__icontains"] = topic_contains + + visible_groups = get_user_visible_groups(user=kwargs["user"]) + + q = ( + queryset.values("group__name", "group__id") + .annotate(count=Count("group__id")) + .filter(**filters) + .filter(group__id__in=[group.group_id for group in visible_groups]) + .order_by("-count") + ) + + logger.debug(" PREFILTERED QUERY ---> %s\n\n", queryset.query) + logger.debug(" ADDITIONAL FILTERS ---> %s\n\n", filters) + logger.debug(" FINAL QUERY ---> %s\n\n", q.query) + + cnt = q.count() + logger.debug(f" q.count() {q.count()}") + logger.info("Found %d facets for %s", cnt, self.name) + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [ + { + "key": r["group__id"], + "label": r["group__name"], + "count": r["count"], + } + for r in q[start:end].all() + ] + + return cnt, topics + + def get_topics(self, keys: list, lang="en", **kwargs) -> list: + q = GroupProfile.objects.filter(group__id__in=keys) + + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + return [{"key": r.group_id, "label": r.slug, "count": len(list(r.resources()))} for r in q.all()] + + @classmethod + def register(cls, registry, **kwargs) -> None: + registry.register_facet_provider(GroupFacetProvider(**kwargs)) diff --git a/geonode/facets/providers/keyword.py b/geonode/facets/providers/keyword.py new file mode 100644 index 0000000..94d9bf2 --- /dev/null +++ b/geonode/facets/providers/keyword.py @@ -0,0 +1,108 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.db.models import Count +from django.utils.translation import gettext_lazy as _ + +from geonode.base.models import HierarchicalKeyword +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, FACET_TYPE_KEYWORD + +logger = logging.getLogger(__name__) + + +class KeywordFacetProvider(FacetProvider): + """ + Implements faceting for resource's keywords + """ + + @property + def name(self) -> str: + return "keyword" + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": self.name, + "filter": "filter{keywords.slug.in}", + "label": _("Keyword"), + "type": FACET_TYPE_KEYWORD, + } + + def get_facet_items( + self, + queryset=None, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for %s", self.name) + + filters = {"resourcebase__in": queryset} + if topic_contains: + filters["name__icontains"] = topic_contains + + if keys: + logger.debug("Filtering by keys %r", keys) + filters["slug__in"] = keys + + q = ( + HierarchicalKeyword.objects.filter(**filters) + .values("slug", "name") + .annotate(count=Count("resourcebase")) + .order_by("-count") + ) + + cnt = q.count() + + logger.info("Found %d facets for %s", cnt, self.name) + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [ + { + "key": r["slug"], + "label": r["name"], + "count": r["count"], + } + for r in q[start:end].all() + ] + + return cnt, topics + + def get_topics(self, keys: list, lang="en", **kwargs) -> list: + q = HierarchicalKeyword.objects.filter(slug__in=keys).values("slug", "name") + + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + return [ + { + "key": r["slug"], + "label": r["name"], + } + for r in q.all() + ] + + @classmethod + def register(cls, registry, **kwargs) -> None: + registry.register_facet_provider(KeywordFacetProvider(**kwargs)) diff --git a/geonode/facets/providers/region.py b/geonode/facets/providers/region.py new file mode 100644 index 0000000..6b59ac0 --- /dev/null +++ b/geonode/facets/providers/region.py @@ -0,0 +1,109 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.db.models import Count +from django.utils.translation import gettext_lazy as _ + +from geonode.base.models import Region +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, FACET_TYPE_PLACE + +logger = logging.getLogger(__name__) + + +class RegionFacetProvider(FacetProvider): + """ + Implements faceting for resource's regions + """ + + @property + def name(self) -> str: + return "region" + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": self.name, + "filter": "filter{regions.code.in}", + "label": _("Region"), + "type": FACET_TYPE_PLACE, + } + + def get_facet_items( + self, + queryset, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for %s", self.name) + + filters = {"resourcebase__in": queryset} + + if topic_contains: + filters["name__icontains"] = topic_contains + + if keys: + logger.debug("Filtering by keys %r", keys) + filters["code__in"] = keys + + q = ( + Region.objects.filter(**filters) + .values("code", "name") + .annotate(count=Count("resourcebase")) + .order_by("-count") + ) + + cnt = q.count() + + logger.info("Found %d facets for %s", cnt, self.name) + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [ + { + "key": r["code"], + "label": r["name"], + "count": r["count"], + } + for r in q[start:end].all() + ] + + return cnt, topics + + def get_topics(self, keys: list, lang="en", **kwargs) -> list: + q = Region.objects.filter(code__in=keys).values("code", "name") + + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + return [ + { + "key": r["code"], + "label": r["name"], + } + for r in q.all() + ] + + @classmethod + def register(cls, registry, **kwargs) -> None: + registry.register_facet_provider(RegionFacetProvider(**kwargs)) diff --git a/geonode/facets/providers/thesaurus.py b/geonode/facets/providers/thesaurus.py new file mode 100644 index 0000000..9262134 --- /dev/null +++ b/geonode/facets/providers/thesaurus.py @@ -0,0 +1,163 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.db.models import Count, OuterRef, Subquery + +from geonode.base.models import ThesaurusKeyword, ThesaurusKeywordLabel +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, FACET_TYPE_THESAURUS + +logger = logging.getLogger(__name__) + + +class ThesaurusFacetProvider(FacetProvider): + """ + Implements faceting for a given Thesaurus + """ + + def __init__(self, identifier, title, order, labels: dict, **kwargs): + super().__init__(**kwargs) + + self._name = identifier + self.label = title + self.labels = labels + + self.config["order"] = order + + @property + def name(self) -> str: + return self._name + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": self._name, + "filter": "filter{tkeywords}", + "label": self.labels.get(lang, self.label), + "is_localized": self.labels.get(lang, None) is not None, + "type": FACET_TYPE_THESAURUS, + } + + def get_facet_items( + self, + queryset, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for %s", self._name) + + filter = {"thesaurus__identifier": self._name, "resourcebase__in": queryset} + + if topic_contains: + filter["label__icontains"] = topic_contains + + if keys: + logger.debug("Filtering by keys %r\n", keys) + filter["id__in"] = keys + + q = ( + ThesaurusKeyword.objects.filter(**filter) + .values("id", "alt_label", "image") + .annotate(count=Count("resourcebase")) + .annotate( + localized_label=Subquery( + ThesaurusKeywordLabel.objects.filter(keyword=OuterRef("id"), lang=lang).values("label") + ) + ) + .order_by("-count") + ) + + logger.debug(" PREFILTERED QUERY ---> %s\n\n", queryset.query) + logger.debug(" ADDITIONAL FILTERS ---> %s\n\n", filter) + logger.debug(" FINAL QUERY ---> %s\n\n", q.query) + + cnt = q.count() + + logger.info("Found %d facets for %s", cnt, self._name) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [ + { + "key": r["id"], + "label": r["localized_label"] or r["alt_label"], + "is_localized": r["localized_label"] is not None, + "count": r["count"], + "image": r["image"], + } + for r in q[start:end].all() + ] + + return cnt, topics + + def get_topics(self, keys: list, lang="en", **kwargs) -> list: + q = ( + ThesaurusKeyword.objects.filter(id__in=keys, thesaurus__identifier=self.name) + .values("id", "alt_label") + .annotate( + localized_label=Subquery( + ThesaurusKeywordLabel.objects.filter(keyword=OuterRef("id"), lang=lang).values("label") + ) + ) + ) + + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + return [ + { + "key": r["id"], + "label": r["localized_label"] or r["alt_label"], + "is_localized": r["localized_label"] is not None, + } + for r in q.all() + ] + + @classmethod + def register(cls, registry, **kwargs) -> None: + # registry.register_facet_provider(CategoryFacetProvider()) + from geonode.base.models import Thesaurus + + # this query return the list of thesaurus X the list of localized titles + q = ( + Thesaurus.objects.filter(facet=True) + .values("identifier", "title", "order", "rel_thesaurus__label", "rel_thesaurus__lang") + .order_by("order") + ) + + # coalesce the localized labels + ret = {} + for r in q.all(): + identifier = r["identifier"] + t = ret.get(identifier, None) + if not t: + t = {k: r[k] for k in ("identifier", "title", "order")} + t["labels"] = {} + if r["rel_thesaurus__lang"] and r["rel_thesaurus__label"]: + t["labels"][r["rel_thesaurus__lang"]] = r["rel_thesaurus__label"] + ret[identifier] = t + + logger.info("Creating providers for %r", ret) + for t in ret.values(): + registry.register_facet_provider( + ThesaurusFacetProvider(t["identifier"], t["title"], t["order"], t["labels"], **kwargs) + ) diff --git a/geonode/facets/providers/users.py b/geonode/facets/providers/users.py new file mode 100644 index 0000000..baa4877 --- /dev/null +++ b/geonode/facets/providers/users.py @@ -0,0 +1,109 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.utils.translation import gettext_lazy as _ +from django.contrib.auth import get_user_model +from django.db.models import Count + +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, FACET_TYPE_USER + +logger = logging.getLogger(__name__) + + +class OwnerFacetProvider(FacetProvider): + """ + Implements faceting for users owner of the resources + """ + + @property + def name(self) -> str: + return "owner" + + def get_info(self, lang="en", **kwargs) -> dict: + return { + "name": "owner", + "filter": "filter{owner.pk.in}", + "label": _("Owner"), + "type": FACET_TYPE_USER, + } + + def get_facet_items( + self, + queryset=None, + start: int = 0, + end: int = DEFAULT_FACET_PAGE_SIZE, + lang="en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ) -> (int, list): + logger.debug("Retrieving facets for OWNER") + + filters = dict() + + if topic_contains: + filters["owner__username__icontains"] = topic_contains + + if keys: + logger.debug("Filtering by keys %r", keys) + filters["owner__in"] = keys + + q = ( + queryset.values("owner", "owner__username") + .filter(**filters) + .annotate(count=Count("owner")) + .order_by("-count") + ) + + cnt = q.count() + + logger.info("Found %d facets for %s", cnt, self.name) + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + topics = [ + { + "key": r["owner"], + "label": r["owner__username"], + "count": r["count"], + } + for r in q[start:end] + ] + + return cnt, topics + + def get_topics(self, keys: list, lang="en", **kwargs) -> list: + q = get_user_model().objects.filter(id__in=keys).values("id", "username") + + logger.debug(" ---> %s\n\n", q.query) + logger.debug(" ---> %r\n\n", q.all()) + + return [ + { + "key": r["id"], + "label": r["username"], + } + for r in q.all() + ] + + @classmethod + def register(cls, registry, **kwargs) -> None: + registry.register_facet_provider(OwnerFacetProvider(**kwargs)) diff --git a/geonode/facets/tests.py b/geonode/facets/tests.py new file mode 100644 index 0000000..5cd6cba --- /dev/null +++ b/geonode/facets/tests.py @@ -0,0 +1,696 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import json +from tastypie.test import TestApiClient +from uuid import uuid4 + +from django.contrib.auth import get_user_model +from django.http import JsonResponse +from django.test import RequestFactory +from django.urls import reverse + +from geonode.base.models import ( + Thesaurus, + ThesaurusLabel, + ThesaurusKeyword, + ThesaurusKeywordLabel, + ResourceBase, + Region, + TopicCategory, + HierarchicalKeyword, + GroupProfile, +) +from geonode.facets.models import facet_registry +from geonode.facets.providers.baseinfo import FeaturedFacetProvider +from geonode.facets.providers.category import CategoryFacetProvider +from geonode.facets.providers.group import GroupFacetProvider +from geonode.facets.providers.keyword import KeywordFacetProvider +from geonode.facets.providers.region import RegionFacetProvider +from geonode.facets.views import ListFacetsView, GetFacetView +from geonode.tests.base import GeoNodeBaseTestSupport +from django.contrib.auth.models import Group + +logger = logging.getLogger(__name__) + + +class TestFacets(GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls): + super().setUpClass() + + cls.user = get_user_model().objects.create(username="user_00") + cls.admin = get_user_model().objects.get(username="admin") + + cls._create_thesauri() + cls._create_regions() + cls._create_categories() + cls._create_keywords() + cls._create_groups() + cls._create_resources() + cls.rf = RequestFactory() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + # remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + + self.api_client = TestApiClient() + + self.assertEqual(self.admin.username, "admin") + self.assertEqual(self.admin.is_superuser, True) + + @classmethod + def _create_thesauri(cls): + cls.thesauri = {} + cls.thesauri_k = {} + + for tn in range(2): + t = Thesaurus.objects.create(identifier=f"t_{tn}", title=f"Thesaurus {tn}", order=100 + tn * 10) + cls.thesauri[tn] = t + for tl in ( # fmt: skip + "en", + "it", + ): + ThesaurusLabel.objects.create(thesaurus=t, lang=tl, label=f"TLabel {tn} {tl}") + + for tkn in range(10): + tk = ThesaurusKeyword.objects.create(thesaurus=t, alt_label=f"T{tn}_K{tkn}_ALT") + cls.thesauri_k[f"{tn}_{tkn}"] = tk + for tkl in ( # fmt: skip + "en", + "it", + ): + ThesaurusKeywordLabel.objects.create(keyword=tk, lang=tkl, label=f"T{tn}_K{tkn}_{tkl}") + + @classmethod + def _create_regions(cls): + cls.regions = {} + + for code, name in ( # fmt: skip + ("R0", "Region0"), + ("R1", "Region1"), + ("R2", "Region2"), + ): + cls.regions[code] = Region.objects.create(code=code, name=name) + + @classmethod + def _create_categories(cls): + cls.cats = {} + + for code, name in ( # fmt: skip + ("C0", "Cat0"), + ("C1", "Cat1"), + ("C2", "Cat2"), + ("C3", "Cat3"), + ): + cls.cats[code] = TopicCategory.objects.create(identifier=code, description=name, gn_description=name) + + @classmethod + def _create_groups(cls): + cls.group_admin = Group.objects.create(name="UserAdmin") + cls.group_common = Group.objects.create(name="UserCommon") + cls.group_profile_admin = GroupProfile.objects.create( + group_id=cls.group_admin, title="UserAdmin", slug="UserAdmin" + ) + cls.group_profile_common = GroupProfile.objects.create( + group_id=cls.group_common, title="UserCommon", slug="UserCommon" + ) + + @classmethod + def _create_keywords(cls): + cls.kw = {} + + for code, name in ( # fmt: skip + ("K0", "Keyword0"), + ("K1", "Keyword1"), + ("K2", "Keyword2"), + ("K3", "Keyword3"), + ): + cls.kw[code] = HierarchicalKeyword.objects.create(slug=code, name=name) + + @classmethod + def _create_resources(self): + public_perm_spec = {"users": {"AnonymousUser": ["view_resourcebase"]}, "groups": []} + for x in range(20): + d: ResourceBase = ResourceBase.objects.create( + title=f"dataset_{x:02}", + uuid=str(uuid4()), + owner=self.user, + abstract=f"Abstract for dataset {x:02}", + subtype="vector", + is_approved=True, + is_published=True, + ) + + # These are the assigned keywords to the Resources + + # RB00 -> T1K0 R0,R1 FEAT K0 C0 group_admin + # RB01 -> T0K0 T1K0 R0 FEAT K1 group_admin + # RB02 -> T1K0 R1 FEAT K2 C0 group_admin + # RB03 -> T0K0 T1K0 K0 group_admin + # RB04 -> T1K0 K0K1 C0 group_admin + # RB05 -> T0K0 T1K0 K0 K2 C1 group_admin + # RB06 -> T1K0 FEAT group_admin + # RB07 -> T0K0 T1K0 R2 FEAT K3 C3 group_common + # RB08 -> T1K0 T1K1 R1,R2 FEAT K3 C3 group_common + # RB09 -> T0K0 T1K0 T1K1 R2 K3 C3 group_common + # RB10 -> T1K1 R2 K3 C3 + # RB11 -> T0K0 T0K1 T1K1 + # RB12 -> T1K1 FEAT + # RB13 -> T0K0 T0K1 R1 FEAT + # RB14 -> FEAT + # RB15 -> T0K0 T0K1 C1 + # RB16 -> C1 + # RB17 -> T0K0 T0K1 + # RB18 -> FEAT C2 + # RB19 -> T0K0 T0K1 FEAT C2 + + if x < 7: + logger.debug(f"ASSIGNING GROUP {self.group_admin.name} TO RB {d}") + d.group = self.group_admin + + if 7 <= x < 10: + logger.debug(f"ASSIGNING GROUP {self.group_common.name} TO RB {d}") + d.group = self.group_common + + if x % 2 == 1: + logger.debug(f"ADDING KEYWORDS {self.thesauri_k['0_0']} to RB {d}") + d.tkeywords.add(self.thesauri_k["0_0"]) + if x % 2 == 1 and x > 10: + logger.debug(f"ADDING KEYWORDS {self.thesauri_k['0_1']} to RB {d}") + d.tkeywords.add(self.thesauri_k["0_1"]) + if x < 10: + logger.debug(f"ADDING KEYWORDS {self.thesauri_k['1_0']} to RB {d}") + d.tkeywords.add(self.thesauri_k["1_0"]) + if 7 < x < 13: + d.tkeywords.add(self.thesauri_k["1_1"]) + + if (x % 6) in (0, 1, 2): + d.featured = True + + for reg, idx in ( # fmt: skip + ("R0", (0, 1)), + ("R1", (0, 2, 8, 13)), + ("R2", (7, 8, 9, 10)), + ): + if x in idx: + d.regions.add(self.regions[reg]) + + for kw, idx in ( # fmt: skip + ("K0", (0, 3, 4, 5)), + ("K1", [1, 4]), + ("K2", [2, 5]), + ("K3", [7, 8, 9, 10]), + ): + if x in idx: + d.keywords.add(self.kw[kw]) + + for cat, idx in ( # fmt: skip + ("C0", [0, 2, 4]), + ("C1", [5, 15, 16]), + ("C2", [18, 19]), + ("C3", [7, 8, 9, 10]), + ): + if x in idx: + d.category = self.cats[cat] + + d.save() + d.set_permissions(public_perm_spec) + + @staticmethod + def _facets_to_map(facets): + return {f["name"]: f for f in facets} + + def test_facets_base(self): + req = self.rf.get(reverse("list_facets"), data={"lang": "en"}) + res: JsonResponse = ListFacetsView.as_view()(req) + obj = json.loads(res.content) + self.assertIn("facets", obj) + facets_list = obj["facets"] + self.assertEqual(9, len(facets_list)) + fmap = self._facets_to_map(facets_list) + for name in ("group", "category", "owner", "t_0", "t_1", "featured", "resourcetype", "keyword"): + self.assertIn(name, fmap) + + def test_facets_rich(self): + # make sure the resources are in + c = ResourceBase.objects.count() + self.assertEqual(20, c) + + # make sure tkeywords have been assigned by checking a sample resource + rb = ResourceBase.objects.get(title="dataset_01") + self.assertEqual(2, rb.tkeywords.count()) + + # run the request + req = self.rf.get(reverse("list_facets"), data={"include_topics": 1, "lang": "en"}) + res: JsonResponse = ListFacetsView.as_view()(req) + obj = json.loads(res.content) + + facets_list = obj["facets"] + self.assertEqual(9, len(facets_list)) + fmap = self._facets_to_map(facets_list) + for expected in ( # fmt: skip + { + "name": "category", + "topics": { + "total": 4, + "items": [ + {"label": "Cat0", "count": 3}, + {"label": "Cat1", "count": 3}, + {"label": "Cat3", "count": 4}, + {"label": "Cat2", "count": 2}, + ], + }, + }, + { + "name": "keyword", + "topics": { + "total": 4, + "items": [ + {"label": "Keyword0", "count": 4}, + {"label": "Keyword1", "count": 2}, + {"label": "Keyword2", "count": 2}, + {"label": "Keyword3", "count": 4}, + ], + }, + }, + { + "name": "owner", + "topics": { + "total": 1, + }, + }, + { + "name": "t_0", + "topics": { + "total": 2, + "items": [ + {"label": "T0_K0_en", "count": 10}, + {"label": "T0_K1_en", "count": 5}, + ], + }, + }, + { + "name": "t_1", + "topics": { + "total": 2, + "items": [ + {"label": "T1_K0_en", "count": 10}, + ], + }, + }, + { + "name": "region", + "topics": { + "total": 3, + "items": [ + {"label": "Region0", "key": "R0", "count": 2}, + {"label": "Region1", "key": "R1", "count": 4}, + {"label": "Region2", "key": "R2", "count": 4}, + ], + }, + }, + { + "name": "featured", + "topics": { + "total": 2, + "items": [ + {"label": "True", "key": True, "count": 11}, + {"label": "False", "key": False, "count": 9}, + ], + }, + }, + { + "name": "resourcetype", + "topics": { + "total": 1, + "items": [ + {"label": "resourcebase", "key": "resourcebase", "count": 20}, + ], + }, + }, + ): + name = expected["name"] + self.assertIn(name, fmap) + facet = fmap[name] + expected_topics = expected["topics"] + for topic_key in expected_topics: + if topic_key != "items": + self.assertEqual( + expected_topics[topic_key], facet["topics"][topic_key], f"Mismatching '{topic_key}' for {name}" + ) + else: + items = facet["topics"]["items"] + expected_items = expected_topics["items"] + for exp_item in expected_items: + exp_label = exp_item["label"] + found = None + for item in items: + if item["label"] == exp_label: + found = item + break + + self.assertIsNotNone( + found, f"topic not found '{exp_label}' for facet '{name}' -- found items {items}" + ) + for exp_field in exp_item: + self.assertEqual( + exp_item[exp_field], found[exp_field], f"Mismatch item key:{exp_field} facet:{name}" + ) + + def test_bad_lang(self): + # for thesauri, make sure that by requesting a non-existent language the faceting is still working, + # using the default labels + + # run the request with a valid language + req = self.rf.get(reverse("get_facet", args=["t_0"]), data={"lang": "en"}) + res: JsonResponse = GetFacetView.as_view()(req, "t_0") + obj = json.loads(res.content) + + self.assertEqual(2, obj["topics"]["total"]) + self.assertEqual(10, obj["topics"]["items"][0]["count"]) + self.assertEqual("T0_K0_en", obj["topics"]["items"][0]["label"]) + self.assertTrue(obj["topics"]["items"][0]["is_localized"]) + + # run the request with an INVALID language + req = self.rf.get(reverse("get_facet", args=["t_0"]), data={"lang": "ZZ"}) + res: JsonResponse = GetFacetView.as_view()(req, "t_0") + obj = json.loads(res.content) + + self.assertEqual(2, obj["topics"]["total"]) + self.assertEqual(10, obj["topics"]["items"][0]["count"]) # make sure the count is still there + self.assertEqual("T0_K0_ALT", obj["topics"]["items"][0]["label"]) # check for the alternate label + self.assertFalse(obj["topics"]["items"][0]["is_localized"]) # check for the localization flag + + def test_prefiltering(self): + reginfo = RegionFacetProvider().get_info() + regfilter = reginfo["filter"] + t0filter = facet_registry.get_provider("t_0").get_info()["filter"] + t1filter = facet_registry.get_provider("t_1").get_info()["filter"] + + for facet, filters, totals, count0 in ( + ("t_0", {}, 2, 10), + ("t_0", {regfilter: "R0"}, 1, 1), + ("t_1", {}, 2, 10), + ("t_1", {regfilter: "R0"}, 1, 2), + ("t_1", {regfilter: "R1"}, 2, 3), + (reginfo["name"], {}, 3, 4), + (reginfo["name"], {t0filter: self.thesauri_k["0_0"].id}, 3, 2), + (reginfo["name"], {t1filter: self.thesauri_k["1_0"].id}, 3, 3), + ): + req = self.rf.get(reverse("get_facet", args=[facet]), data=filters) + res: JsonResponse = GetFacetView.as_view()(req, facet) + obj = json.loads(res.content) + self.assertEqual( + totals, + obj["topics"]["total"], + f"Bad totals for facet '{facet} and filter {filters}\nRESPONSE: {obj}", + ) + self.assertEqual( + count0, obj["topics"]["items"][0]["count"], f"Bad count0 for facet '{facet}\nRESPONSE: {obj}" + ) + + def test_prefiltering_tkeywords(self): + regname = RegionFacetProvider().name + featname = FeaturedFacetProvider().name + t1filter = facet_registry.get_provider("t_1").get_info()["filter"] + tkey_1_1 = self.thesauri_k["1_1"].id + + expected_region = {"R1": 1, "R2": 3} + expected_feat = {True: 2, False: 3} + + # Run the single requests + for facet, params, items in ( + (regname, {t1filter: tkey_1_1}, expected_region), + (featname, {t1filter: tkey_1_1}, expected_feat), + ): + req = self.rf.get(reverse("get_facet", args=[facet]), data=params) + res: JsonResponse = GetFacetView.as_view()(req, facet) + obj = json.loads(res.content) + + self.assertEqual( + len(items), + len(obj["topics"]["items"]), + f"Bad count for items '{facet} \n PARAMS: {params} \n RESULT: {obj} \n EXPECTED: {items}", + ) + # search item + for item in items.keys(): + found = next((i for i in obj["topics"]["items"] if i["key"] == item), None) + self.assertIsNotNone(found, f"Topic '{item}' not found in facet {facet} -- {obj}") + self.assertEqual(items[item], found.get("count", None), f"Bad count for facet '{facet}:{item}") + + # Run the single request + req = self.rf.get(reverse("list_facets"), data={"include_topics": 1, t1filter: tkey_1_1}) + res: JsonResponse = ListFacetsView.as_view()(req) + obj = json.loads(res.content) + + facets_list = obj["facets"] + fmap = self._facets_to_map(facets_list) + + for name, items in ( # fmt: skip + (regname, expected_region), + (featname, expected_feat), + ): + self.assertIn(name, fmap) + facet = fmap[name] + + for item in items.keys(): + found = next((i for i in facet["topics"]["items"] if i["key"] == item), None) + self.assertIsNotNone(found, f"Topic '{item}' not found in facet {facet} -- {facet}") + self.assertEqual(items[item], found.get("count", None), f"Bad count for facet '{facet}:{item}") + + def test_config(self): + for facet, type, order in ( + ("resourcetype", None, None), + ("t_0", "select", 100), + ("category", "select", 5), + ("region", "select", 7), + ("owner", "select", 8), + ): + req = self.rf.get(reverse("get_facet", args=[facet]), data={"include_config": True}) + res: JsonResponse = GetFacetView.as_view()(req, facet) + obj = json.loads(res.content) + self.assertIn("config", obj, "Config info not found in payload") + conf = obj["config"] + + if type is None: + self.assertNotIn("type", conf) + else: + self.assertEqual(type, conf["type"], "Unexpected type") + + if order is None: + self.assertNotIn("order", conf) + else: + self.assertEqual(order, conf["order"], "Unexpected order") + + def test_count0(self): + reginfo = RegionFacetProvider().get_info() + regflt = reginfo["filter"] + regname = reginfo["name"] + + catinfo = CategoryFacetProvider().get_info() + catflt = catinfo["filter"] + catname = catinfo["name"] + + kwinfo = KeywordFacetProvider().get_info() + kwflt = kwinfo["filter"] + kwname = kwinfo["name"] + + groupinfo = GroupFacetProvider().get_info() + grflt = groupinfo["filter"] + grname = groupinfo["name"] + g_admin_id = self.group_admin.id + g_comm_id = self.group_common.id + + t0flt = facet_registry.get_provider("t_0").get_info()["filter"] + t1flt = facet_registry.get_provider("t_1").get_info()["filter"] + + def t(tk): + return self.thesauri_k[tk].id + + for facet, params, items in ( # fmt: skip + # thesauri + ("t_1", {regflt: "R0"}, {t("1_0"): 2}), + ("t_1", {regflt: "R0", "key": [t("1_0")]}, {t("1_0"): 2}), + ("t_1", {regflt: "R0", t0flt: t("0_1")}, {}), + ("t_1", {regflt: "R0", t0flt: t("0_1"), "key": [t("1_0")]}, {t("1_0"): None}), + ( + "t_1", + {regflt: "R0", t0flt: t("0_1"), "key": [t("1_1"), t("1_0")]}, + {t("1_0"): None, t("1_1"): None}, + ), + ("t_1", {"key": [t("0_1")]}, {}), + ("t_1", {t0flt: t("0_0")}, {t("1_0"): 5, t("1_1"): 2}), + ("t_1", {t0flt: t("0_1")}, {t("1_1"): 1}), + ("t_1", {t0flt: [t("0_1"), t("0_0")]}, {t("1_0"): 5, t("1_1"): 2}), + ("t_1", {catflt: ["C0"]}, {t("1_0"): 3}), + ("t_1", {catflt: ["C0", "C1"]}, {t("1_0"): 4}), + # regions + (regname, {t1flt: t("1_0")}, {"R0": 2, "R1": 3, "R2": 3}), + (regname, {t1flt: t("1_1")}, {"R1": 1, "R2": 3}), + (regname, {t1flt: [t("1_1"), t("1_0")]}, {"R0": 2, "R1": 3, "R2": 4}), + (regname, {t1flt: t("1_1"), "key": ["R0", "R1"]}, {"R1": 1, "R0": None}), + (regname, {t1flt: t("1_1"), "key": ["R0"]}, {"R0": None}), + # groups + (grname, {grflt: [g_admin_id, g_comm_id], "key": [g_admin_id, g_comm_id]}, {g_admin_id: 7, g_comm_id: 3}), + (grname, {grflt: [g_admin_id], "key": [g_admin_id]}, {g_admin_id: 7}), + (grname, {catflt: ["C0"], grflt: [g_comm_id]}, {}), + (grname, {catflt: ["C1"], grflt: [g_admin_id]}, {g_admin_id: 1}), + (grname, {catflt: ["C0"], grflt: [g_admin_id]}, {g_admin_id: 3}), + (grname, {catflt: ["C0", "C1"], grflt: [g_admin_id, g_comm_id]}, {g_admin_id: 4}), + # category + (catname, {t1flt: t("1_0")}, {"C0": 3, "C1": 1, "C3": 3}), + (catname, {t1flt: t("1_0"), "key": ["C0", "C2"]}, {"C0": 3, "C2": None}), + (catname, {t1flt: [t("1_0"), t("1_1")]}, {"C0": 3, "C1": 1, "C3": 4}), + (catname, {kwflt: "K1"}, {"C0": 1}), + (catname, {kwflt: "K1", "key": ["C0", "C2"]}, {"C0": 1, "C2": None}), + # keyword + (kwname, {t0flt: t("0_0")}, {"K0": 2, "K1": 1, "K2": 1, "K3": 2}), + (kwname, {t0flt: t("1_0")}, {"K0": 4, "K1": 2, "K2": 2, "K3": 3}), + (kwname, {t0flt: [t("1_0"), t("1_1")]}, {"K0": 4, "K1": 2, "K2": 2, "K3": 4}), + (kwname, {t0flt: t("0_0"), regflt: "R0"}, {"K1": 1}), + (kwname, {t0flt: t("0_0"), regflt: "R0", "key": ["K0"]}, {"K0": None}), + ): + req = self.rf.get(reverse("get_facet", args=[facet]), data=params) + req.user = self.admin + res: JsonResponse = GetFacetView.as_view()(req, facet) + obj = json.loads(res.content) + # self.assertEqual(totals, obj["topics"]["total"], f"Bad totals for facet '{facet} and params {params}") + + self.assertEqual( + len(items), + len(obj["topics"]["items"]), + f"Bad count for items '{facet} \n PARAMS: {params} \n RESULT: {obj} \n EXPECTED: {items}", + ) + # search item + for item in items.keys(): + found = next((i for i in obj["topics"]["items"] if i["key"] == item), None) + self.assertIsNotNone(found, f"Topic '{item}' not found in facet {facet} -- {obj}") + self.assertEqual(items[item], found.get("count", None), f"Bad count for facet '{facet}:{item}") + + def test_user_auth(self): + # make sure the user authorization pre-filters the visible resources + # TODO test + pass + + def test_thesauri_reloading(self): + # Thesauri facets are cached. + # Make sure that when Thesauri or ThesauriLabel change the facets cache is invalidated + # TODO impl+test + + pass + + def test_group_facet_api_call(self): + resource_count_admin = 7 + resource_count_common = 3 + + expected_response_base = { + "name": "group", + "filter": "filter{group.in}", + "label": "Group", + "type": "group", + "topics": { + "page": 0, + "page_size": 10, + "start": 0, + "total": 2, + "items": [ + { + "key": self.group_profile_admin.group_id, + "label": self.group_profile_admin.slug, + "count": resource_count_admin, + }, + { + "key": self.group_profile_common.group_id, + "label": self.group_profile_common.slug, + "count": resource_count_common, + }, + ], + }, + } + expected_response_filtered = { + "name": "group", + "filter": "filter{group.in}", + "label": "Group", + "type": "group", + "topics": { + "page": 0, + "page_size": 10, + "start": 0, + "total": 1, + "items": [ + { + "key": self.group_profile_admin.group_id, + "label": self.group_profile_admin.slug, + "count": resource_count_admin, + } + ], + }, + } + + url_filtered = f"{reverse('get_facet',args=['group'])}?filter{{group.in}}={self.group_admin.id}&include_topics=true&key={self.group_admin.id}" + url_base = f"{reverse('get_facet',args=['group'])}" + + response_filtered = self.client.get(url_filtered) + response_dict_filtered = response_filtered.json() + + response_base = self.client.get(url_base) + response_dict_base = response_base.json() + + self.assertEqual( + response_filtered.status_code, + 200, + "Unexpected status code, got %s expected 200" % (response_filtered.status_code), + ) + self.assertEqual( + response_base.status_code, 200, "Unexpected status code, got %s expected 200" % (response_base.status_code) + ) + + self.assertDictEqual(expected_response_filtered, response_dict_filtered) + self.assertDictEqual(expected_response_base, response_dict_base) + + def test_group_facets_are_filtered_by_words(self): + # there are some groups and the facets return them + url = f"{reverse('get_facet',args=['group'])}" + + response = self.client.get(url) + self.assertEqual(200, response.status_code, response.json()) + + self.assertTrue(response.json().get("topics", {}).get("total", 0) > 0) + + # topic_contains with real name should return 1 + url = f"{reverse('get_facet',args=['group'])}?topic_contains=UserAdmin" + response = self.client.get(url) + + self.assertEqual(200, response.status_code, response.json()) + + self.assertEqual(1, response.json().get("topics", {}).get("total", 0)) + + # topic_contains with a random string to be searched for should be 0 + url = f"{reverse('get_facet',args=['group'])}?topic_contains=abc123scfuqbrwefbasascgiu" + response = self.client.get(url) + + self.assertEqual(200, response.status_code, response.json()) + + self.assertEqual(0, response.json().get("topics", {}).get("total", 0)) diff --git a/geonode/facets/urls.py b/geonode/facets/urls.py new file mode 100644 index 0000000..8d96976 --- /dev/null +++ b/geonode/facets/urls.py @@ -0,0 +1,26 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.urls import path +from .views import ListFacetsView, GetFacetView + +urlpatterns = [ + path("facets", ListFacetsView.as_view(), name="list_facets"), + path("facets/", GetFacetView.as_view(), name="get_facet"), +] diff --git a/geonode/facets/views.py b/geonode/facets/views.py new file mode 100644 index 0000000..d4cabc6 --- /dev/null +++ b/geonode/facets/views.py @@ -0,0 +1,221 @@ +######################################################################### +# +# Copyright (C) 2023 Open Source Geospatial Foundation - all rights reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +from urllib.parse import urlencode + +from rest_framework.authentication import SessionAuthentication, BasicAuthentication +from rest_framework.views import APIView + +from django.http import HttpResponseNotFound, JsonResponse +from django.urls import reverse +from django.conf import settings + +from geonode.base.api.views import ResourceBaseViewSet +from geonode.base.models import ResourceBase +from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, facet_registry +from geonode.security.utils import get_visible_resources + +PARAM_PAGE = "page" +PARAM_PAGE_SIZE = "page_size" +PARAM_LANG = "lang" +PARAM_ADD_LINKS = "add_links" +PARAM_INCLUDE_TOPICS = "include_topics" +PARAM_INCLUDE_CONFIG = "include_config" +PARAM_TOPIC_CONTAINS = "topic_contains" + +logger = logging.getLogger(__name__) + + +class BaseFacetingView(APIView): + authentication_classes = [SessionAuthentication, BasicAuthentication] + + @classmethod + def _get_topics( + cls, + provider, + queryset, + page: int = 0, + page_size: int = DEFAULT_FACET_PAGE_SIZE, + lang: str = "en", + topic_contains: str = None, + keys: set = {}, + **kwargs, + ): + start = page * page_size + end = start + page_size + + cnt, items = provider.get_facet_items( + queryset, start=start, end=end, lang=lang, topic_contains=topic_contains, keys=keys, **kwargs + ) + + if keys: + keys.difference_update({str(t["key"]) for t in items}) + if keys: + ext = provider.get_topics(keys, lang) + items.extend(ext) + cnt += len(ext) + logger.debug("Extending facets to %d for %s", cnt, provider.name) + + return {"page": page, "page_size": page_size, "start": start, "total": cnt, "items": items} + + @classmethod + def _prefilter_topics(cls, request): + """ + Perform some prefiltering on resources, such as + - auth visibility + - filtering by other facets already applied + :param request: + :return: a QuerySet on ResourceBase + """ + logger.debug("Filtering by user '%s'", request.user) + filters = {k: vlist for k, vlist in request.query_params.lists() if k.startswith("filter{")} + logger.warning(f"FILTERING BY {filters}") + + if filters: + viewset = ResourceBaseViewSet(request=request, format_kwarg={}, kwargs=filters) + viewset.initial(request) + return get_visible_resources(queryset=viewset.filter_queryset(viewset.get_queryset()), user=request.user) + else: + # return ResourceBase.objects + return get_visible_resources(ResourceBase.objects, request.user) + + @classmethod + def _resolve_language(cls, request) -> (str, bool): + """ + :return: the resolved language, a boolean telling if the language was requested + """ + # first try with an explicit request using params + if lang := request.GET.get(PARAM_LANG, None): + return lang, True + # 2nd try: use LANGUAGE_CODE + try: + return request.LANGUAGE_CODE.split("-")[0], False + except AttributeError: + return settings.LANGUAGE_CODE, False + + @classmethod + def _resolve_boolean(cls, request, name, fallback=None): + """ + Parse boolean query params + """ + val = request.GET.get(name, None) + if val is None: + return fallback + + val = val.lower() + if val.startswith("t") or val.startswith("y") or val == "1": + return True + elif val.startswith("f") or val.startswith("n") or val == "0": + return False + else: + return fallback + + +class ListFacetsView(BaseFacetingView): + def get(self, request, *args, **kwargs): + lang, lang_requested = self._resolve_language(request) + add_links = self._resolve_boolean(request, PARAM_ADD_LINKS, False) + include_topics = self._resolve_boolean(request, PARAM_INCLUDE_TOPICS, False) + include_config = self._resolve_boolean(request, PARAM_INCLUDE_CONFIG, False) + + facets = [] + prefiltered = None + + for provider in facet_registry.get_providers(): + logger.debug("Fetching data from provider %r", provider) + info = provider.get_info(lang=lang) + + if include_config: + info["config"] = provider.config + + if add_links: + link_args = {PARAM_ADD_LINKS: True} + if lang_requested: # only add lang param if specified in current call + link_args[PARAM_LANG] = lang + info["link"] = f"{reverse('get_facet', args=[info['name']])}?{urlencode(link_args)}" + + if include_topics: + prefiltered = prefiltered or self._prefilter_topics(request) + info["topics"] = self._get_topics(provider, queryset=prefiltered, lang=lang, user=request.user) + + facets.append(info) + + logger.debug("Returning facets %r", facets) + return JsonResponse({"facets": facets}) + + +class GetFacetView(BaseFacetingView): + def get(self, request, facet): + logger.debug("get_facet -> %r for user '%r'", facet, request.user.username) + + # retrieve provider for the requested facet + provider: FacetProvider = facet_registry.get_provider(facet) + if not provider: + return HttpResponseNotFound() + + # parse some query params + lang, lang_requested = self._resolve_language(request) + add_link = self._resolve_boolean(request, PARAM_ADD_LINKS, False) + include_config = self._resolve_boolean(request, PARAM_INCLUDE_CONFIG, False) + + topic_contains = request.GET.get(PARAM_TOPIC_CONTAINS, None) + keys = set(request.query_params.getlist("key")) + + page = int(request.GET.get(PARAM_PAGE, 0)) + page_size = int(request.GET.get(PARAM_PAGE_SIZE, DEFAULT_FACET_PAGE_SIZE)) + + info = provider.get_info(lang) + if include_config: + info["config"] = provider.config + + qs = self._prefilter_topics(request) + topics = self._get_topics( + provider, + queryset=qs, + page=page, + page_size=page_size, + lang=lang, + topic_contains=topic_contains, + keys=keys, + user=request.user, + ) + + if add_link: + exist_prev = page > 0 + exist_next = topics["total"] > (page + 1) * page_size + link = reverse("get_facet", args=[info["name"]]) + for exist, link_name, p in ( + (exist_prev, "prev", page - 1), + (exist_next, "next", page + 1), + ): + link_param = {PARAM_PAGE: p, PARAM_PAGE_SIZE: page_size, PARAM_LANG: lang, PARAM_ADD_LINKS: True} + if lang_requested: # only add lang param if specified in current call + link_param[PARAM_LANG] = lang + if topic_contains: + link_param[PARAM_TOPIC_CONTAINS] = topic_contains + info[link_name] = f"{link}?{urlencode(link_param)}" if exist else None + + if topic_contains: + # in the payload let's rmb this is a filtered output + info["topic_contains"] = topic_contains + + info["topics"] = topics + + return JsonResponse(info) diff --git a/geonode/favorite/__init__.py b/geonode/favorite/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/favorite/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/favorite/apps.py b/geonode/favorite/apps.py new file mode 100644 index 0000000..0b27c6e --- /dev/null +++ b/geonode/favorite/apps.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + + +class GeoNodeFavoriteAppConfig(AppConfig): + name = "geonode.favorite" + verbose_name = "GeoNode Favorite" diff --git a/geonode/favorite/migrations/0001_initial.py b/geonode/favorite/migrations/0001_initial.py new file mode 100644 index 0000000..e622872 --- /dev/null +++ b/geonode/favorite/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 1.11.20 on 2019-05-23 10:36 + + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Favorite', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField()), + ('created_on', models.DateTimeField(auto_now_add=True)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'favorite', + 'verbose_name_plural': 'favorites', + }, + ), + migrations.AlterUniqueTogether( + name='favorite', + unique_together={('user', 'content_type', 'object_id')}, + ), + ] diff --git a/geonode/favorite/migrations/__init__.py b/geonode/favorite/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/favorite/models.py b/geonode/favorite/models.py new file mode 100644 index 0000000..fe7b717 --- /dev/null +++ b/geonode/favorite/models.py @@ -0,0 +1,114 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.fields import GenericForeignKey +from django.db import models + +from geonode.base.models import ResourceBase +from geonode.documents.models import Document +from geonode.layers.models import Dataset +from geonode.maps.models import Map +from geonode.utils import get_geonode_app_types + + +class FavoriteManager(models.Manager): + def favorites_for_user(self, user): + return self.filter(user=user) + + def _favorite_ct_for_user(self, user, model): + content_type = ContentType.objects.get_for_model(model) + result = self.favorites_for_user(user).filter(content_type=content_type).prefetch_related("content_object") + return result + + def favorite_documents_for_user(self, user): + return self._favorite_ct_for_user(user, Document) + + def favorite_maps_for_user(self, user): + return self._favorite_ct_for_user(user, Map) + + def favorite_datasets_for_user(self, user): + return self._favorite_ct_for_user(user, Dataset) + + def favorite_users_for_user(self, user): + return self._favorite_ct_for_user(user, get_user_model()) + + def favorite_for_user_and_content_object(self, user, content_object): + """ + if Favorite exists for input user and type and pk of the input + content_object, return it. else return None. + impl note: can only be 0 or 1, per the class's unique_together. + """ + content_type = ContentType.objects.get_for_model(type(content_object)) + result = self.filter(user=user, content_type=content_type, object_id=content_object.pk) + + if result and len(result) > 0: + return result[0] + else: + return None + + def bulk_favorite_objects(self, user): + "get the actual favorite objects for a user as a dict by content_type" + favs = {} + for m in (Document, Map, Dataset, get_user_model()): + ct = ContentType.objects.get_for_model(m) + f = self.favorites_for_user(user).filter(content_type=ct) + favs[ct.name] = m.objects.filter(id__in=f.values("object_id")) + return favs + + def create_favorite(self, content_object, user): + if isinstance(content_object, ResourceBase): + geoapp_types = get_geonode_app_types() + if content_object.resource_type in geoapp_types: + content_type = ContentType.objects.get(model="geoapp") + else: + content_type = ContentType.objects.filter(model=content_object.resource_type).last() + else: + content_type = ContentType.objects.get_for_model(type(content_object)) + + favorite, _ = self.get_or_create( + user=user, + content_type=content_type, + object_id=content_object.pk, + ) + return favorite + + +class Favorite(models.Model): + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey("content_type", "object_id") + + created_on = models.DateTimeField(auto_now_add=True) + + objects = FavoriteManager() + + class Meta: + verbose_name = "favorite" + verbose_name_plural = "favorites" + unique_together = (("user", "content_type", "object_id"),) + + def __str__(self): + if self.content_object: + return f"Favorite: {self.content_object.title}, {self.content_type}, {self.user}" + else: + return "Unknown" diff --git a/geonode/favorite/tests.py b/geonode/favorite/tests.py new file mode 100644 index 0000000..efaceef --- /dev/null +++ b/geonode/favorite/tests.py @@ -0,0 +1,124 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.base.models import ResourceBase +from geonode.tests.base import GeoNodeBaseTestSupport + +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType +from django.urls import reverse + +from .models import Favorite +from geonode.documents.models import Document +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset + + +class FavoriteTest(GeoNodeBaseTestSupport): + type = "document" + + """ + Tests geonode.favorite app/module + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.adm_un = "admin" + self.adm_pw = "admin" + + # tests of Favorite and FavoriteManager methods. + def test_favorite(self): + # assume we created at least one User and two Documents in setUp. + test_user = get_user_model().objects.first() + test_document_1 = Document.objects.first() + test_document_2 = Document.objects.last() + self.assertIsNotNone(test_document_1) + self.assertIsNotNone(test_document_2) + + # test create favorite. + Favorite.objects.create_favorite(test_document_1, test_user) + Favorite.objects.create_favorite(test_document_2, test_user) + favorites = Favorite.objects.all() + self.assertEqual(favorites.count(), 2) + ct = ContentType.objects.get_for_model(test_document_1) + self.assertEqual(favorites[0].content_type, ct) + + # test all favorites for specific user. + favorites_for_user = Favorite.objects.favorites_for_user(test_user) + self.assertEqual(favorites_for_user.count(), 2) + + # test document favorites for user. + document_favorites = Favorite.objects.favorite_documents_for_user(test_user) + self.assertEqual(document_favorites.count(), 2) + + # test layer favorites for user. + dataset_favorites = Favorite.objects.favorite_datasets_for_user(test_user) + self.assertEqual(dataset_favorites.count(), 0) + + # test map favorites for user. + map_favorites = Favorite.objects.favorite_maps_for_user(test_user) + self.assertEqual(map_favorites.count(), 0) + + # test user favorites for user. + user_favorites = Favorite.objects.favorite_users_for_user(test_user) + self.assertEqual(user_favorites.count(), 0) + + # test favorite for user and a specific content object. + user_content_favorite = Favorite.objects.favorite_for_user_and_content_object(test_user, test_document_1) + self.assertEqual(user_content_favorite.object_id, test_document_1.id) + + # test bulk favorites. + bulk_favorites = Favorite.objects.bulk_favorite_objects(test_user) + self.assertEqual(len(bulk_favorites[ct.name]), 2) + self.assertEqual(len(bulk_favorites["user"]), 0) + + def test_given_resource_base_object_will_assign_subtype_as_content_type(self): + test_user = get_user_model().objects.first() + + """ + If the input object is a ResourceBase, in favorite content type, should be saved he + subtype content type (Doc, Dataset, Map or GeoApp) + """ + create_single_dataset("foo_dataset") + resource = ResourceBase.objects.get(title="foo_dataset") + created_fav = Favorite.objects.create_favorite(resource, test_user) + self.assertEqual("dataset", created_fav.content_type.model) + + """ + If the input object is a subtype, should save the relative content type + """ + test_document_1 = Document.objects.first() + self.assertIsNotNone(test_document_1) + Favorite.objects.create_favorite(test_document_1, test_user) + fav = Favorite.objects.last() + ct = ContentType.objects.get_for_model(test_document_1) + self.assertEqual(fav.content_type, ct) + + def _get_response(self, input_url, input_args): + return self.client.post(reverse(input_url, args=input_args), content_type="application/json") diff --git a/geonode/favorite/utils.py b/geonode/favorite/utils.py new file mode 100644 index 0000000..985b1ed --- /dev/null +++ b/geonode/favorite/utils.py @@ -0,0 +1,44 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.urls import reverse +from . import models + + +def get_favorite_info(user, content_object): + """ + return favorite info dict containing: + a. an add favorite url for the input parameters. + b. whether there is an existing Favorite for the input parameters. + c. a delete url (if there is an existing Favorite). + """ + result = {} + + url_content_type = type(content_object).__name__.lower() + result["add_url"] = reverse(f"add_favorite_{url_content_type}", args=[content_object.pk]) + + existing_favorite = models.Favorite.objects.favorite_for_user_and_content_object(user, content_object) + + if existing_favorite: + result["has_favorite"] = "true" + result["delete_url"] = reverse("delete_favorite", args=[existing_favorite.pk]) + else: + result["has_favorite"] = "false" + + return result diff --git a/geonode/geoapps/__init__.py b/geonode/geoapps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/geoapps/admin.py b/geonode/geoapps/admin.py new file mode 100644 index 0000000..33c8ff2 --- /dev/null +++ b/geonode/geoapps/admin.py @@ -0,0 +1,83 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib import admin + +from modeltranslation.admin import TabbedTranslationAdmin + +from geonode.geoapps.models import GeoApp +from geonode.base.admin import ResourceBaseAdminForm + + +class GeoAppAdminForm(ResourceBaseAdminForm): + class Meta(ResourceBaseAdminForm.Meta): + model = GeoApp + fields = "__all__" + + +class GeoAppAdmin(TabbedTranslationAdmin): + exclude = ("ll_bbox_polygon", "bbox_polygon", "srid") + list_display_links = ("title",) + list_display = ( + "id", + "title", + "type", + "owner", + "category", + "group", + "is_approved", + "is_published", + ) + list_editable = ( + "owner", + "category", + "group", + "is_approved", + "is_published", + ) + list_filter = ( + "title", + "owner", + "category", + "group", + "is_approved", + "is_published", + ) + search_fields = ( + "title", + "abstract", + "purpose", + "is_approved", + "is_published", + ) + readonly_fields = ("geographic_bounding_box",) + form = GeoAppAdminForm + + def delete_queryset(self, request, queryset): + """ + We need to invoke the 'ResourceBase.delete' method even when deleting + through the admin batch action + """ + for obj in queryset: + from geonode.resource.manager import resource_manager + + resource_manager.delete(obj.uuid, instance=obj) + + +admin.site.register(GeoApp, GeoAppAdmin) diff --git a/geonode/geoapps/api/__init__.py b/geonode/geoapps/api/__init__.py new file mode 100644 index 0000000..0044807 --- /dev/null +++ b/geonode/geoapps/api/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoapps/api/exceptions.py b/geonode/geoapps/api/exceptions.py new file mode 100644 index 0000000..fc8b686 --- /dev/null +++ b/geonode/geoapps/api/exceptions.py @@ -0,0 +1,40 @@ +######################################################################### +# +# Copyright (C) 2022 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from rest_framework.exceptions import APIException + + +class DuplicateGeoAppException(APIException): + status_code = 409 + default_detail = "GeoApp already exists" + default_code = "geoapp_exception" + category = "geoapp_api" + + +class InvalidGeoAppException(APIException): + status_code = 400 + default_detail = "The provided data is not valid" + default_code = "geoapp_exception" + category = "geoapp_api" + + +class GeneralGeoAppException(APIException): + status_code = 500 + default_detail = "An error has occurred while processing your request" + default_code = "geoapp_exception" + category = "geoapp_api" diff --git a/geonode/geoapps/api/permissions.py b/geonode/geoapps/api/permissions.py new file mode 100644 index 0000000..1c48cbe --- /dev/null +++ b/geonode/geoapps/api/permissions.py @@ -0,0 +1,55 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.conf import settings +from rest_framework.filters import BaseFilterBackend + +from geonode.geoapps.models import GeoApp + + +class GeoAppPermissionsFilter(BaseFilterBackend): + """ + A filter backend that limits results to those where the requesting user + has read object level permissions. + """ + + shortcut_kwargs = { + "accept_global_perms": True, + } + + def filter_queryset(self, request, queryset, view): + # We want to defer this import until runtime, rather than import-time. + # See https://github.com/encode/django-rest-framework/issues/4608 + # (Also see #1624 for why we need to make this import explicitly) + from guardian.shortcuts import get_objects_for_user + from geonode.security.utils import get_visible_resources + + user = request.user + resources = get_objects_for_user(user, "base.view_resourcebase", **self.shortcut_kwargs) + + _allowed_ids = get_visible_resources( + resources, + user, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + ).values_list("id", flat=True) + + obj_with_perms = [_app.id for _app in GeoApp.objects.filter(id__in=_allowed_ids)] + + return queryset.filter(id__in=obj_with_perms) diff --git a/geonode/geoapps/api/serializers.py b/geonode/geoapps/api/serializers.py new file mode 100644 index 0000000..c89440c --- /dev/null +++ b/geonode/geoapps/api/serializers.py @@ -0,0 +1,123 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.contrib.auth import get_user_model +from geonode.geoapps.api.exceptions import DuplicateGeoAppException, InvalidGeoAppException, GeneralGeoAppException + +from geonode.geoapps.models import GeoApp +from geonode.resource.manager import resource_manager +from geonode.base.api.serializers import ResourceBaseSerializer + +logger = logging.getLogger(__name__) + + +class GeoAppSerializer(ResourceBaseSerializer): + def __init__(self, *args, **kwargs): + # Instantiate the superclass normally + super().__init__(*args, **kwargs) + + class Meta: + model = GeoApp + name = "geoapp" + view_name = "geoapps-list" + fields = list(set(ResourceBaseSerializer.Meta.fields + ("name",))) + + def extra_update_checks(self, validated_data): + _user_profiles = {} + for _key, _value in validated_data.items(): + if _key in ("owner", "poc", "metadata_owner"): + _user_profiles[_key] = _value + for _key, _value in _user_profiles.items(): + validated_data.pop(_key) + _u = get_user_model().objects.filter(username=_value).first() + if _u: + validated_data[_key] = _u + else: + raise InvalidGeoAppException(f"The specified '{_key}' does not exist!") + + def extra_create_checks(self, validated_data): + if "name" not in validated_data or "owner" not in validated_data: + raise InvalidGeoAppException("No valid data: 'name' and 'owner' are mandatory fields!") + + if self.Meta.model.objects.filter(name=validated_data["name"]).count(): + raise DuplicateGeoAppException("A GeoApp with the same 'name' already exists!") + + self.extra_update_checks(validated_data) + + def _sanitize_validated_data(self, validated_data, instance=None): + # Extract users' profiles + _user_profiles = {} + for _key, _value in validated_data.items(): + if _key in ("owner", "poc", "metadata_owner"): + _user_profiles[_key] = _value + for _key, _value in _user_profiles.items(): + validated_data.pop(_key) + _u = get_user_model().objects.filter(username=_value).first() + if _u: + validated_data[_key] = _u + else: + raise InvalidGeoAppException(f"The specified '{_key}' does not exist!") + + return validated_data + + def _create_and_update(self, validated_data, instance=None): + # Extract JSON blob + _data = {} + if "blob" in validated_data: + _data = validated_data.pop("blob") + + # Create a new instance + if not instance: + _instance = resource_manager.create(None, resource_type=self.Meta.model, defaults=validated_data) + else: + _instance = instance + + try: + self.Meta.model.objects.filter(pk=_instance.id).update(**validated_data) + _instance.refresh_from_db() + except Exception as e: + raise GeneralGeoAppException(e) + + # Let's finalize the instance and notify the users + validated_data["blob"] = _data + return resource_manager.update(_instance.uuid, instance=_instance, vals=validated_data, notify=True) + + def create(self, validated_data): + # Sanity checks + _mandatory_fields = ["name", "owner", "resource_type"] + if not all([_x in validated_data for _x in _mandatory_fields]): + raise InvalidGeoAppException(f"No valid data: {_mandatory_fields} are mandatory fields!") + + if self.Meta.model.objects.filter(name=validated_data["name"]).count(): + raise DuplicateGeoAppException("A GeoApp with the same 'name' already exists!") + + return self._create_and_update(self._sanitize_validated_data(validated_data)) + + def update(self, instance, validated_data): + if instance.resource_type: + validated_data["resource_type"] = instance.resource_type + # Sanity checks + _mandatory_fields = [ + "resource_type", + ] + if not all([_x in validated_data for _x in _mandatory_fields]): + raise InvalidGeoAppException(f"No valid data: {_mandatory_fields} are mandatory fields!") + + return self._create_and_update(self._sanitize_validated_data(validated_data), instance=instance) diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py new file mode 100644 index 0000000..fc40909 --- /dev/null +++ b/geonode/geoapps/api/tests.py @@ -0,0 +1,285 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import json +import logging +from unittest.mock import MagicMock +from urllib.parse import urljoin + +from django.contrib.auth import get_user_model +from django.urls import reverse +from rest_framework.test import APITestCase + +from geonode.base.populate_test_data import create_models +from geonode.geoapps.api.exceptions import DuplicateGeoAppException, InvalidGeoAppException +from geonode.geoapps.api.serializers import GeoAppSerializer +from geonode.geoapps.models import GeoApp + +logger = logging.getLogger(__name__) + + +class GeoAppsApiTests(APITestCase): + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + def setUp(self): + create_models(b"document") + create_models(b"map") + create_models(b"dataset") + self.admin = get_user_model().objects.get(username="admin") + self.bobby = get_user_model().objects.get(username="bobby") + self.norman = get_user_model().objects.get(username="norman") + self.gep_app = GeoApp.objects.create( + title="Test GeoApp", + owner=self.bobby, + resource_type="geostory", + blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}', + ) + self.gep_app.set_default_permissions() + + def test_geoapps_list(self): + """ + Ensure we can access the GeoApps list. + """ + url = reverse("geoapps-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 1) + # Pagination + self.assertEqual(len(response.data["geoapps"]), 1) + self.assertTrue("data" not in response.data["geoapps"][0]) + + response = self.client.get(f"{url}?include[]=data", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 1) + # Pagination + self.assertEqual(len(response.data["geoapps"]), 1) + self.assertTrue("data" in response.data["geoapps"][0]) + self.assertEqual( + json.loads(response.data["geoapps"][0]["data"]), {"test_data": {"test": ["test_1", "test_2", "test_3"]}} + ) + + def test_geoapp_listing_advertised(self): + app = GeoApp.objects.first() + app.advertised = False + app.save() + + url = reverse("geoapps-list") + + payload = self.client.get(url) + + prev_count = payload.json().get("total") + # the user can see only the advertised resources + self.assertEqual(GeoApp.objects.filter(advertised=True).count(), prev_count) + + payload = self.client.get(f"{url}?advertised=True") + # so if advertised is True, we dont see the advertised=False resource + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count) + + payload = self.client.get(f"{url}?advertised=False") + # so if advertised is False, we see only the resource with advertised==False + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, 1) + + # if all is requested, we will see all the resources + payload = self.client.get(f"{url}?advertised=all") + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count + 1) + + GeoApp.objects.update(advertised=True) + + def test_extra_metadata_included_with_param(self): + _app = GeoApp.objects.first() + url = urljoin(f"{reverse('geoapps-list')}/", f"{_app.pk}") + data = {"include[]": "metadata"} + + response = self.client.get(url, format="json", data=data) + self.assertIsNotNone(response.data["geoapp"].get("metadata")) + + response = self.client.get(url, format="json") + self.assertNotIn("metadata", response.data["geoapp"]) + + def test_geoapps_crud(self): + """ + Ensure we can create/update GeoApps. + """ + # Bobby + self.assertTrue(self.client.login(username="bobby", password="bob")) + # Create + url = f"{reverse('geoapps-list')}?include[]=data" + data = { + "name": "Test Create", + "title": "Test Create", + "resource_type": "geostory", + "owner": "bobby", + "extent": {"coords": [1123692.0, 5338214.0, 1339852.0, 5482615.0], "srid": "EPSG:3857"}, + } + response = self.client.post(url, data=data, format="json") + self.assertEqual(response.status_code, 201) # 201 - Created + + x = GeoApp.objects.filter(title="Test Create").first() + self.assertEqual(x.srid, "EPSG:3857") + self.assertEqual(response.json()["geoapp"].get("extent")["srid"], "EPSG:4326") + self.assertEqual( + response.json()["geoapp"].get("extent")["coords"], + [10.094296982428332, 43.1721654049465, 12.03609530058109, 44.11086592050112], + ) + + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 2) + # Pagination + self.assertEqual(len(response.data["geoapps"]), 2) + + # Update: PATCH + url = reverse("geoapps-detail", kwargs={"pk": self.gep_app.pk}) + data = {"blob": {"test_data": {"test": ["test_4", "test_5", "test_6"]}}} + response = self.client.patch(url, data=json.dumps(data), content_type="application/json") + self.assertEqual(response.status_code, 200) + + response = self.client.get(f"{url}?include[]=data", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + # Pagination + self.assertTrue("data" in response.data["geoapp"]) + self.assertEqual(response.data["geoapp"]["resource_type"], "geostory") + self.assertEqual(response.data["geoapp"]["data"], {"test_data": {"test": ["test_4", "test_5", "test_6"]}}) + + # Update: POST + data = {"test_data": {"test": ["test_1", "test_2", "test_3"]}} + + _app = GeoApp.objects.first() + _app.set_permissions({"users": {self.bobby: ["base.add_resourcebase", "base.delete_resourcebase"]}}) + + response = self.client.post(url, data=json.dumps(data), format="json") + self.assertEqual(response.status_code, 405) # 405 – Method not allowed + + # Delete + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, 405) # 405 - Method Not Allowed + + response = self.client.get(f"{url}?include[]=data", format="json") + self.assertEqual(response.status_code, 200) + + def test_extra_create_checks_with_no_owner(self): + serializer = GeoAppSerializer() + data = {"name": "fakename"} + with self.assertRaises(InvalidGeoAppException) as exp: + serializer.extra_create_checks(data) + + self.assertEqual(exp.exception.category, "geoapp_api") + self.assertEqual(exp.exception.default_code, "geoapp_exception") + self.assertEqual(str(exp.exception.detail), "No valid data: 'name' and 'owner' are mandatory fields!") + + def test_extra_create_checks_duplicated(self): + serializer = GeoAppSerializer() + _geoapp = GeoApp.objects.first() + data = {"name": _geoapp.name, "owner": _geoapp.owner} + with self.assertRaises(DuplicateGeoAppException) as exp: + serializer.extra_create_checks(data) + + self.assertEqual(exp.exception.category, "geoapp_api") + self.assertEqual(exp.exception.default_code, "geoapp_exception") + self.assertEqual(str(exp.exception.detail), "A GeoApp with the same 'name' already exists!") + + def test_create_with_no_owner(self): + serializer = GeoAppSerializer() + data = {"name": "fakename"} + with self.assertRaises(InvalidGeoAppException) as exp: + serializer.create(data) + + self.assertEqual(exp.exception.category, "geoapp_api") + self.assertEqual(exp.exception.default_code, "geoapp_exception") + self.assertEqual( + str(exp.exception.detail), "No valid data: ['name', 'owner', 'resource_type'] are mandatory fields!" + ) + + def test_update_with_no_owner(self): + serializer = GeoAppSerializer() + data = {"name": "fakename"} + _geoapp = MagicMock() + _geoapp.resource_type = None + with self.assertRaises(InvalidGeoAppException) as exp: + serializer.update(_geoapp, data) + + self.assertEqual(exp.exception.category, "geoapp_api") + self.assertEqual(exp.exception.default_code, "geoapp_exception") + self.assertEqual(str(exp.exception.detail), "No valid data: ['resource_type'] are mandatory fields!") + + def test_create_checks_duplicated(self): + serializer = GeoAppSerializer() + _geoapp = GeoApp.objects.first() + data = {"name": _geoapp.name, "owner": _geoapp.owner, "resource_type": _geoapp.resource_type} + with self.assertRaises(DuplicateGeoAppException) as exp: + serializer.create(data) + + self.assertEqual(exp.exception.category, "geoapp_api") + self.assertEqual(exp.exception.default_code, "geoapp_exception") + self.assertEqual(str(exp.exception.detail), "A GeoApp with the same 'name' already exists!") + + def test_geoapp_creation_owner_is_mantained(self): + """ + https://github.com/GeoNode/geonode/issues/12261 + The geoapp owner should be mantained even if another + user save the instance. + """ + + url = f"{reverse('geoapps-list')}?include[]=data" + data = { + "name": "Test Create", + "title": "Test Create", + "resource_type": "geostory", + "extent": {"coords": [1123692.0, 5338214.0, 1339852.0, 5482615.0], "srid": "EPSG:3857"}, + } + + self.assertTrue(self.client.login(username="norman", password="norman")) + + response = self.client.post(url, data=data, format="json") + + self.assertEqual(201, response.status_code) + # let's check that the owner is the request one + self.assertEqual("norman", response.json()["geoapp"]["owner"]["username"]) + + # let's change the user of the request + self.assertTrue(self.client.login(username="admin", password="admin")) + + sut = GeoApp.objects.get(pk=response.json()["geoapp"]["pk"]) + # Update: PATCH + # we ensure that norman is the resource owner + self.assertEqual("norman", sut.owner.username) + + url = reverse("geoapps-detail", kwargs={"pk": sut.pk}) + data = {"blob": {"test_data": {"test": ["test_4", "test_5", "test_6"]}}} + # sending the update of the geoapp + response = self.client.patch(url, data=json.dumps(data), content_type="application/json") + + self.assertEqual(response.status_code, 200) + + # ensure that the value of the owner is not changed + sut.refresh_from_db() + self.assertEqual("norman", sut.owner.username) + + # cleanup + sut.delete() diff --git a/geonode/geoapps/api/urls.py b/geonode/geoapps/api/urls.py new file mode 100644 index 0000000..a39cde0 --- /dev/null +++ b/geonode/geoapps/api/urls.py @@ -0,0 +1,25 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.api.urls import router + +from . import views + +router.register(r"geoapps", views.GeoAppViewSet, "geoapps") + +urlpatterns = [] diff --git a/geonode/geoapps/api/views.py b/geonode/geoapps/api/views.py new file mode 100644 index 0000000..57136b9 --- /dev/null +++ b/geonode/geoapps/api/views.py @@ -0,0 +1,69 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from dynamic_rest.viewsets import DynamicModelViewSet +from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter + +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.authentication import SessionAuthentication, BasicAuthentication +from oauth2_provider.contrib.rest_framework import OAuth2Authentication + +from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter +from geonode.base.api.mixins import AdvertisedListMixin +from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.base.api.permissions import UserHasPerms +from geonode.base.api.views import ApiPresetsInitializer +from geonode.geoapps.models import GeoApp + +from .serializers import GeoAppSerializer +from .permissions import GeoAppPermissionsFilter + +import logging + +logger = logging.getLogger(__name__) + + +class GeoAppViewSet(ApiPresetsInitializer, DynamicModelViewSet, AdvertisedListMixin): + """ + API endpoint that allows geoapps to be viewed or edited. + """ + + http_method_names = ["get", "patch", "post", "put"] + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [ + IsAuthenticatedOrReadOnly, + UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}), + ] + filter_backends = [ + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + ExtentFilter, + GeoAppPermissionsFilter, + ] + queryset = GeoApp.objects.all().order_by("-created") + serializer_class = GeoAppSerializer + pagination_class = GeoNodeApiPagination + + def perform_create(self, serializer): + """ + The owner is not passed from the FE + so we force the request.user to be the owner + in creation + """ + return serializer.save(owner=self.request.user) diff --git a/geonode/geoapps/apps.py b/geonode/geoapps/apps.py new file mode 100644 index 0000000..3985e34 --- /dev/null +++ b/geonode/geoapps/apps.py @@ -0,0 +1,59 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig +from django.utils.translation import gettext_noop as _ +from geonode.notifications_helper import NotificationsAppConfigBase + + +class GeoNodeAppsConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.geoapps" + type = "GEONODE_APP" + + NOTIFICATIONS = ( + ( + "geoapp_created", + _("App Created"), + _("A App was created"), + ), + ( + "geoapp_updated", + _("App Updated"), + _("A App was updated"), + ), + ( + "geoapp_approved", + _("App Approved"), + _("A App was approved by a Manager"), + ), + ( + "geoapp_published", + _("App Published"), + _("A App was published"), + ), + ( + "geoapp_deleted", + _("App Deleted"), + _("A App was deleted"), + ), + ( + "geoapp_rated", + _("Rating for App"), + _("A rating was given to an App"), + ), + ) diff --git a/geonode/geoapps/forms.py b/geonode/geoapps/forms.py new file mode 100644 index 0000000..818f391 --- /dev/null +++ b/geonode/geoapps/forms.py @@ -0,0 +1,45 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.geoapps.models import GeoApp +from geonode.base.forms import ResourceBaseForm, get_tree_data + + +class GeoAppForm(ResourceBaseForm): + class Meta(ResourceBaseForm.Meta): + model = GeoApp + exclude = ResourceBaseForm.Meta.exclude + ("zoom", "projection", "center_x", "center_y", "data") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["regions"].choices = get_tree_data() + for field in self.fields: + help_text = self.fields[field].help_text + self.fields[field].help_text = None + if help_text != "": + self.fields[field].widget.attrs.update( + { + "class": "has-external-popover", + "data-content": help_text, + "placeholder": help_text, + "data-placement": "right", + "data-container": "body", + "data-html": "true", + } + ) diff --git a/geonode/geoapps/migrations/0001_initial.py b/geonode/geoapps/migrations/0001_initial.py new file mode 100644 index 0000000..972b519 --- /dev/null +++ b/geonode/geoapps/migrations/0001_initial.py @@ -0,0 +1,48 @@ +# Generated by Django 2.2.15 on 2020-10-13 12:48 + +from django.db import migrations, models +import django.db.models.deletion +from django.db.models.fields.json import JSONField + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('base', '0045_auto_20200507_0445'), + ] + + operations = [ + migrations.CreateModel( + name='GeoApp', + fields=[ + ('resourcebase_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='base.ResourceBase')), + ('name', models.TextField(db_index=True, unique=True, verbose_name='Name')), + ('zoom', models.IntegerField(blank=True, null=True, verbose_name='zoom')), + ('projection', models.CharField(blank=True, max_length=32, null=True, verbose_name='projection')), + ('center_x', models.FloatField(blank=True, null=True, verbose_name='center X')), + ('center_y', models.FloatField(blank=True, null=True, verbose_name='center Y')), + ('last_modified', models.DateTimeField(auto_now_add=True)), + ('urlsuffix', models.CharField(blank=True, max_length=255, null=True, verbose_name='Site URL')), + ], + options={ + 'abstract': False, + 'base_manager_name': 'objects', + }, + bases=('base.resourcebase',), + ), + migrations.CreateModel( + name='GeoAppData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('blob', JSONField(default=dict)), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='geoapps.GeoApp')), + ], + ), + migrations.AddField( + model_name='geoapp', + name='data', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='data', to='geoapps.GeoAppData'), + ), + ] diff --git a/geonode/geoapps/migrations/0002_auto_20210615_0717.py b/geonode/geoapps/migrations/0002_auto_20210615_0717.py new file mode 100644 index 0000000..5371b8f --- /dev/null +++ b/geonode/geoapps/migrations/0002_auto_20210615_0717.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2 on 2021-06-15 07:17 + +from django.db import migrations + + +move_data_blob = ''' +UPDATE base_resourcebase +SET "blob"=subquery."blob" +FROM (select resource_id,"blob" from geoapps_geoappdata gg) AS subquery +WHERE base_resourcebase.id=subquery.resource_id; +''' + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0001_initial'), + ('base', '0066_resourcebase_data'), + ] + + operations = [ + migrations.RunSQL(move_data_blob), + migrations.RemoveField( + model_name='geoapp', + name='data', + ), + migrations.DeleteModel( + name='GeoAppData', + ), + ] diff --git a/geonode/geoapps/migrations/0003_alter_geoapp_name.py b/geonode/geoapps/migrations/0003_alter_geoapp_name.py new file mode 100644 index 0000000..8d13b7d --- /dev/null +++ b/geonode/geoapps/migrations/0003_alter_geoapp_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-06-22 14:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0002_auto_20210615_0717'), + ] + + operations = [ + migrations.AlterField( + model_name='geoapp', + name='name', + field=models.TextField(verbose_name='Name'), + ), + ] diff --git a/geonode/geoapps/migrations/0003_update_detail_url.py b/geonode/geoapps/migrations/0003_update_detail_url.py new file mode 100644 index 0000000..ee2d071 --- /dev/null +++ b/geonode/geoapps/migrations/0003_update_detail_url.py @@ -0,0 +1,17 @@ +from django.db import migrations +from django.db.migrations.operations import RunPython + + +def update_geoapps_detail_url(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0002_auto_20210615_0717'), + ] + + operations = [ + RunPython(update_geoapps_detail_url, migrations.RunPython.noop) + ] diff --git a/geonode/geoapps/migrations/0004_merge_0003_alter_geoapp_name_0003_update_detail_url.py b/geonode/geoapps/migrations/0004_merge_0003_alter_geoapp_name_0003_update_detail_url.py new file mode 100644 index 0000000..1530716 --- /dev/null +++ b/geonode/geoapps/migrations/0004_merge_0003_alter_geoapp_name_0003_update_detail_url.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.4 on 2021-06-23 08:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0003_alter_geoapp_name'), + ('geoapps', '0003_update_detail_url'), + ] + + operations = [ + ] diff --git a/geonode/geoapps/migrations/0005_auto_20210705_1121.py b/geonode/geoapps/migrations/0005_auto_20210705_1121.py new file mode 100644 index 0000000..501c06b --- /dev/null +++ b/geonode/geoapps/migrations/0005_auto_20210705_1121.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.4 on 2021-07-05 11:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0004_merge_0003_alter_geoapp_name_0003_update_detail_url'), + ] + + operations = [ + migrations.RemoveField( + model_name='geoapp', + name='center_x', + ), + migrations.RemoveField( + model_name='geoapp', + name='center_y', + ), + migrations.RemoveField( + model_name='geoapp', + name='projection', + ), + migrations.RemoveField( + model_name='geoapp', + name='urlsuffix', + ), + migrations.RemoveField( + model_name='geoapp', + name='zoom', + ), + ] diff --git a/geonode/geoapps/migrations/0006_geoapp_blob_migration.py b/geonode/geoapps/migrations/0006_geoapp_blob_migration.py new file mode 100644 index 0000000..3fe0800 --- /dev/null +++ b/geonode/geoapps/migrations/0006_geoapp_blob_migration.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.4 on 2021-07-05 11:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0005_auto_20210705_1121'), + ] + + operations = [ + ] diff --git a/geonode/geoapps/migrations/__init__.py b/geonode/geoapps/migrations/__init__.py new file mode 100644 index 0000000..4799694 --- /dev/null +++ b/geonode/geoapps/migrations/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### \ No newline at end of file diff --git a/geonode/geoapps/models.py b/geonode/geoapps/models.py new file mode 100644 index 0000000..30e6ef7 --- /dev/null +++ b/geonode/geoapps/models.py @@ -0,0 +1,99 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.db import models +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ + +from guardian.shortcuts import get_anonymous_user + +from geonode.base.models import ResourceBase +from geonode.client.hooks import hookset + +logger = logging.getLogger("geonode.geoapps.models") + + +class GeoApp(ResourceBase): + """ + A GeoApp it is a generic container for every client applications the + user might want to create or define. + """ + + PERMISSIONS = { + "write": [ + "change_geoapp_data", + "change_geoapp_style", + ] + } + + name = models.TextField(_("Name"), null=False, blank=False) + + last_modified = models.DateTimeField(auto_now_add=True) + # The last time the geoapp was modified. + + def __str__(self): + return f'{self.title} by {(self.owner.username if self.owner else "")}' + + @property + def class_name(self): + return self.__class__.__name__ + + @property + def sender(self): + return None + + @property + def center(self): + """ + A handy shortcut for the center_x and center_y properties as a tuple + (read only) + """ + return (self.center_x, self.center_y) + + @property + def type(self): + _ct = self.polymorphic_ctype + _child = _ct.model_class().objects.filter(pk=self.id).first() + if _child and hasattr(_child, "app_type"): + return _child.app_type + return None + + @property + def is_public(self): + """ + Returns True if anonymous (public) user can view geoapp. + """ + user = get_anonymous_user() + return user.has_perm("base.view_resourcebase", obj=self.resourcebase_ptr) + + @property + def keywords_list(self): + keywords_qs = self.keywords.all() + if keywords_qs: + return [kw.name for kw in keywords_qs] + else: + return [] + + def get_absolute_url(self): + return hookset.geoapp_detail_url(self) + + @property + def embed_url(self): + return reverse("geoapp_embed", kwargs={"geoappid": self.pk}) diff --git a/geonode/geoapps/templates/apps/app_edit.html b/geonode/geoapps/templates/apps/app_edit.html new file mode 100644 index 0000000..8af81fc --- /dev/null +++ b/geonode/geoapps/templates/apps/app_edit.html @@ -0,0 +1,20 @@ +{% extends "geonode_base.html" %} + +{% load i18n %} +{% load base_tags %} +{% load client_lib_tags %} + +{% block title %} {% trans "Edit" %} {% blocktrans %}{{GEONODE_APPS_NAME}}{% endblocktrans %} - {{ block.super }} {% endblock %} +{% block head %} + + + + {% get_geoapp_edit %} + {{ block.super }} +{% endblock %} + +{% block footer %}{% endblock %} diff --git a/geonode/geoapps/templates/apps/app_embed.html b/geonode/geoapps/templates/apps/app_embed.html new file mode 100644 index 0000000..066fde4 --- /dev/null +++ b/geonode/geoapps/templates/apps/app_embed.html @@ -0,0 +1,14 @@ +{% load i18n %} +{% load base_tags %} +{% load client_lib_tags %} + +{% block head %} + + + + {% get_geoapp_embed %} +{% endblock %} \ No newline at end of file diff --git a/geonode/geoapps/templates/apps/app_metadata.html b/geonode/geoapps/templates/apps/app_metadata.html new file mode 100644 index 0000000..c195b43 --- /dev/null +++ b/geonode/geoapps/templates/apps/app_metadata.html @@ -0,0 +1,87 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load bootstrap_tags %} +{% load base_tags %} +{% load guardian_tags %} +{% load floppyforms %} + +{% block title %}{{ geoapp.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock body_class %} + +{% block body_outer %} + +
      +

      {% trans "Metadata" %} {% blocktrans with geoapp.title as map_title %}for {{ map_title }}{% endblocktrans %}

      +
      + {% trans "Completeness" %} +
      + + +
      +
      +
      + 70% +
      +
      +
      +
      + +
      + {% if geoapp.metadata_uploaded %} +
      {% blocktrans %}Note: this geoapp's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}
      + {% endif %} + + {% if geoapp_form.errors or category_form.errors or tkeywords_form.errors %} +
      {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %} +
        + {% for field in geoapp_form %} + {% if field.errors %} +
      • {{ field.label }}
        • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
      • {{ category_form.errors.as_ul }}
        • + {% endif %} + {% if tkeywords_form.errors %} +
      • {{ tkeywords_form.errors.as_ul }}
        • + {% endif %} +
      +
      + {% endif %} + + {% csrf_token %} +
      + {% form geoapp_form using panel_template %} + {# geoapp_form|as_bootstrap #} +
      + +
      +
      + + + +
      + + + >" %}"/> +
      +
      +
      +
      + + + +{{ block.super }} +{% endblock body_outer %} diff --git a/geonode/geoapps/templates/apps/app_metadata_advanced.html b/geonode/geoapps/templates/apps/app_metadata_advanced.html new file mode 100644 index 0000000..b6c0d7b --- /dev/null +++ b/geonode/geoapps/templates/apps/app_metadata_advanced.html @@ -0,0 +1,137 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load static %} +{% load base_tags %} +{% load bootstrap_tags %} +{% load guardian_tags %} +{% load client_lib_tags %} +{% block title %}{{ geoapp.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock %} + +{% block body_outer %} + +{{ block.super }} + + + + + + + + + + +
      + {% trans "Explore GeoApps" %} +

      {% trans "Edit Metadata" %}

      +
      +
      +
      +

      + {% blocktrans with geoapp.title as map_title %} + Editing details for {{ map_title }} + {% endblocktrans %} +

      + +
      + {% if geoapp.metadata_uploaded %} +
      {% blocktrans %}Note: this geoapp's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}
      + {% endif %} + + {% if geoapp_form.errors or category_form.errors %} +
      {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %} +
        + {% for field in geoapp_form %} + {% if field.errors %} +
      • {{ field.label }}
        • + {% endif %} + {% endfor %} + + {% if category_form.errors %} +
      • {{ category_form.errors.as_ul }}
        • + {% endif %} +
      +
      + {% endif %} +
      + {% trans "Return to" %} {% trans resource.resource_type %} + +
      + {% csrf_token %} + +
      + {% block geoapp_fields %} + {% for field in geoapp_form %} + {% if field.name != 'use_featureinfo_custom_template' and field.name != 'featureinfo_custom_template' and field.name not in ADVANCED_EDIT_EXCLUDE_FIELD %} + {% if field.name == 'featured' and not user.is_superuser %} + {% else %} +
      +
      + + {{ field }} +
      +
      + {% endif %} + {% endif %} + {% endfor %} + {% endblock geoapp_fields %} + + + {% block thesauri %} + {% if THESAURI_FILTERS %} + {% for field in tkeywords_form %} +
      +

      + + {{ field }} +

      +
      + {% endfor %} + {% endif %} + {% endblock thesauri %} +
      +
      +
      + +
      + {% autoescape off %} + {% for choice in category_form.category_choice_field.field.choices %} +
      + +
      + {% endfor %} + {% endautoescape %} +
      +
      + +
      + + + +
      + {% trans "Return to" %} {% trans resource.resource_type %} + +
      +
      +
      +
      +
      +
      +{% endblock %} diff --git a/geonode/geoapps/templates/apps/app_metadata_detail.html b/geonode/geoapps/templates/apps/app_metadata_detail.html new file mode 100644 index 0000000..83004ed --- /dev/null +++ b/geonode/geoapps/templates/apps/app_metadata_detail.html @@ -0,0 +1,6 @@ +{% extends "metadata_detail.html" %} +{% load i18n %} +{% block metaget_absolute_url %} +
      {% trans "Metadata Page" %}
      +
      {% url "geoapp_metadata_detail" resource.id %}
      +{% endblock metaget_absolute_url %} \ No newline at end of file diff --git a/geonode/geoapps/templates/apps/app_new.html b/geonode/geoapps/templates/apps/app_new.html new file mode 100644 index 0000000..e672271 --- /dev/null +++ b/geonode/geoapps/templates/apps/app_new.html @@ -0,0 +1,20 @@ +{% extends "geonode_base.html" %} + +{% load i18n %} +{% load base_tags %} +{% load client_lib_tags %} + +{% block title %} {% trans "New Map" %} - {{ block.super }} {% endblock %} +{% block head %} + + + + {% get_geoapp_new %} + {{ block.super }} +{% endblock %} + +{% block footer %}{% endblock %} diff --git a/geonode/geoapps/templates/layouts/app_panels.html b/geonode/geoapps/templates/layouts/app_panels.html new file mode 100644 index 0000000..23ff9b5 --- /dev/null +++ b/geonode/geoapps/templates/layouts/app_panels.html @@ -0,0 +1,608 @@ +{% load i18n %} +{% load static %} +{% load floppyforms %} +{% load contact_roles %} + + + + + + + + + + + + + + + + + + +{% block body_outer %} + + +
      +
      +
      + +
      + {% trans "Mandatory" %} +
      +
      + {% trans "Mandatory" %} +
      +
      + {% trans "Optional" %} +
      +
      + +
      + +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + + +
      +
      + {% block geoapp_title %} + + + {{ geoapp_form.title }} + {% endblock %} +
      + + + {{ geoapp_form.abstract }} +
      +
      +
      +
      + + + {{ geoapp_form.date_type }} +
      +
      + + + {{ geoapp_form.date }} +
      + {% block geoapp_category %} +
      + + +
      + {% endblock geoapp_category %} +
      + + +
      +
      + + {{ geoapp_form.keywords }} +
      + {% if THESAURI_FILTERS %} +
      + {{tkeywords_form.as_p}} +
      + {% endif %} +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      +
      + {% block geoapp_attributes %} +
      +
      + + + {{ geoapp_form.language }} +
      +
      + + + {{ geoapp_form.license }} +
      +
      + + {{ geoapp_form.attribution }} +
      +
      + {% endblock geoapp_attributes %} +
      +
      + + {{ geoapp_form.regions }} +
      +
      + + + {{ geoapp_form.data_quality_statement }} +
      +
      +
      +
      + + + {{ geoapp_form.restriction_code_type }} +
      +
      + + + {{ geoapp_form.constraints_other }} +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +

      {% trans "Other, Optional, Metadata" %}

      +
      + + + {{ geoapp_form.edition }} +
      +
      + + {{ geoapp_form.doi }} +
      +
      + + + {{ geoapp_form.purpose }} +
      +
      + + + {{ geoapp_form.supplemental_information }} +
      +
      +
      + {% block geoapp_temporal_extent_start %} +
      +
      + + + {{ geoapp_form.temporal_extent_start }} +
      +
      + {% endblock geoapp_temporal_extent_start %} + {% block geoapp_temporal_extent_end %} +
      +
      + + + {{ geoapp_form.temporal_extent_end }} +
      +
      + {% endblock geoapp_temporal_extent_end %} + {% block maintenance_block %} +
      +
      + + + {{ geoapp_form.maintenance_frequency }} +
      +
      + + + {{ geoapp_form.spatial_representation_type }} +
      + {% block geoapp_extra_metadata %} +
      + + {{ geoapp_form.extra_metadata }} +
      + {% endblock geoapp_extra_metadata %} + {% block geoapp_linked_resources %} +
      + + {{ geoapp_form.linked_resources }} +
      + {% endblock geoapp_linked_resources %} + +
      + {% endblock maintenance_block %} +
      +
      + {% block geoapp_poc %} +
      +
      {% trans "Responsible Parties" %}
      +
      + + {{ geoapp_form.poc }} +
      +
      + {% endblock %} +
      +
      {% trans "Responsible and Permissions" %}
      + {% block geoapp_owner %} +
      +
      + + {{ geoapp_form.owner }} +
      + {% endblock geoapp_owner %} +
      +
      + {% trans "toggle more Contact Roles" %} + {% block geoapp_more_contact_roles %} +
      +
      {% trans "more metadata contact roles" %}
      + {% for contact_role in UI_ROLES_IN_TOGGLE_VIEW %} + {% getattribute geoapp_form contact_role as cr %} +
      +
      + + {{ cr}} +
      +
      + {% endfor %} +
      +
      + {% endblock geoapp_more_contact_roles %} +
      + +
      +
      +
      + {% block extra_metadata_content %} + {% endblock %} +
      +
      +
      + +
      +
      +
      +
      +
      +
      {% trans "Publishing" %}
      +
      +
      + + {{ geoapp_form.metadata_uploaded_preserve }} +
      +
      + + {{ geoapp_form.is_approved }} +
      +
      + + {{ geoapp_form.is_published }} +
      + {% if user.is_superuser %} +
      + + {{ geoapp_form.featured }} +
      + {% endif %} +
      + + {{ geoapp_form.advertised }} +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      {% trans "Other Settings" %}
      +
      +
      + + {{ geoapp_form.id }} +
      +
      +
      +
      +
      +
      +
      +
      +
      +{% endblock %} diff --git a/geonode/geoapps/tests.py b/geonode/geoapps/tests.py new file mode 100644 index 0000000..c4566d6 --- /dev/null +++ b/geonode/geoapps/tests.py @@ -0,0 +1,186 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.urls import reverse +from django.test import override_settings +from django.contrib.auth import get_user_model + +from geonode.geoapps.models import GeoApp +from geonode.geoapps.forms import GeoAppForm +from geonode.base.models import TopicCategory +from geonode.resource.manager import resource_manager +from geonode.tests.base import GeoNodeBaseTestSupport + +from geonode.base.populate_test_data import all_public, create_models, remove_models + + +class GeoAppTests(GeoNodeBaseTestSupport): + """Tests geonode.geoapps module""" + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.bobby = get_user_model().objects.get(username="bobby") + self.geo_app = resource_manager.create( + None, + resource_type=GeoApp, + defaults=dict( + title="Testing GeoApp", owner=self.bobby, blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}' + ), + ) + self.user = get_user_model().objects.get(username="admin") + self.geoapp = GeoApp.objects.create( + name="name", title="geoapp_titlte", thumbnail_url="initial", owner=self.user + ) + self.sut = GeoAppForm + + def test_resource_form_is_invalid_extra_metadata_not_json_format(self): + self.client.login(username="admin", password="admin") + url = reverse("geoapp_metadata", args=(self.geoapp.id,)) + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "not-a-json", + }, + ) + expected = { + "success": False, + "errors": ["extra_metadata: The value provided for the Extra metadata field is not a valid JSON"], + } + self.assertDictEqual(expected, response.json()) + + @override_settings(EXTRA_METADATA_SCHEMA={"key": "value"}) + def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): + self.client.login(username="admin", password="admin") + url = reverse("geoapp_metadata", args=(self.geoapp.id,)) + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "[{'key': 'value'}]", + }, + ) + expected = { + "success": False, + "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource geoapp"], + } + self.assertDictEqual(expected, response.json()) + + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): + self.client.login(username="admin", password="admin") + url = reverse("geoapp_metadata", args=(self.geoapp.id,)) + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": '[{"key": "value"},{"id": "int", "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + ) + expected = ( + "extra_metadata: Missing keys: 'field_label', 'field_name', 'field_value', 'filter_header' at index 0 " + ) + self.assertIn(expected, response.json()["errors"][0]) + + @override_settings( + EXTRA_METADATA_SCHEMA={ + "geoapp": { + "id": int, + "filter_header": object, + "field_name": object, + "field_label": object, + "field_value": object, + } + } + ) + def test_resource_form_is_valid_extra_metadata(self): + form = self.sut( + data={ + "owner": self.geoapp.owner.id, + "title": "geoapp_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + user=self.user, + ) + self.assertTrue(form.is_valid()) + + def test_geoapp_category_is_correctly_assigned_in_metadata_upload(self): + self.client.login(username="admin", password="admin") + url = reverse("geoapp_metadata", args=(self.geoapp.id,)) + + # assign a category to the GeoApp + category = TopicCategory.objects.order_by("identifier").first() + self.geoapp.category = category + self.geoapp.save() + # retrieving the new one + new_category = TopicCategory.objects.order_by("identifier").last() + + response = self.client.post( + url, + data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "category_choice_field": new_category.id, + }, + ) + self.geoapp.refresh_from_db() + self.assertEqual(200, response.status_code) + self.assertEqual(new_category.identifier, self.geoapp.category.identifier) + + def test_geoapp_copy(self): + self.client.login(username="admin", password="admin") + geoapp_copy = None + try: + geoapp_copy = resource_manager.copy(self.geoapp, defaults=dict(title="Testing GeoApp 2")) + self.assertIsNotNone(geoapp_copy) + self.assertEqual(geoapp_copy.title, "Testing GeoApp 2") + finally: + if geoapp_copy: + geoapp_copy.delete() + self.assertIsNotNone(self.geoapp) diff --git a/geonode/geoapps/translation.py b/geonode/geoapps/translation.py new file mode 100644 index 0000000..c06694f --- /dev/null +++ b/geonode/geoapps/translation.py @@ -0,0 +1,28 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from modeltranslation.translator import translator, TranslationOptions +from geonode.geoapps.models import GeoApp + + +class GeoAppTranslationOptions(TranslationOptions): + fields = () + + +translator.register(GeoApp, GeoAppTranslationOptions) diff --git a/geonode/geoapps/urls.py b/geonode/geoapps/urls.py new file mode 100644 index 0000000..16f4c7e --- /dev/null +++ b/geonode/geoapps/urls.py @@ -0,0 +1,37 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.urls import include, re_path + +from . import views + +js_info_dict = { + "packages": ("geonode.geoapps",), +} + +urlpatterns = [ + # 'geonode.geoapps.views', + re_path(r"^new$", views.new_geoapp, name="new_geoapp"), + re_path(r"^(?P\d+)/metadata$", views.geoapp_metadata, name="geoapp_metadata"), + re_path(r"^(?P[^/]*)/metadata_detail$", views.geoapp_metadata_detail, name="geoapp_metadata_detail"), + re_path(r"^(?P\d+)/metadata_advanced$", views.geoapp_metadata_advanced, name="geoapp_metadata_advanced"), + re_path( + r"^(?P[^/]+)/embed$", views.geoapp_edit, {"template": "apps/app_embed.html"}, name="geoapp_embed" + ), + re_path(r"^", include("geonode.geoapps.api.urls")), +] diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py new file mode 100644 index 0000000..a20064f --- /dev/null +++ b/geonode/geoapps/views.py @@ -0,0 +1,399 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import ast +import json +import logging +import warnings +import traceback + +from django.conf import settings +from django.shortcuts import render +from django.utils.translation import gettext_lazy as _ +from django.contrib.auth.decorators import login_required +from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.core.exceptions import PermissionDenied, ObjectDoesNotExist +from django.views.decorators.clickjacking import xframe_options_sameorigin +from geonode.base.enumerations import SOURCE_TYPE_LOCAL + +from geonode.client.hooks import hookset +from geonode.people.forms import ProfileForm +from geonode.base import register_event +from geonode.groups.models import GroupProfile +from geonode.monitoring.models import EventType +from geonode.base.auth import get_or_create_token +from geonode.security.views import _perms_info_json +from geonode.security.utils import get_user_visible_groups +from geonode.geoapps.models import GeoApp +from geonode.resource.manager import resource_manager +from geonode.decorators import check_keyword_write_perms + +from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm +from geonode.base.models import Thesaurus, TopicCategory +from geonode.utils import resolve_object + +from .forms import GeoAppForm + +logger = logging.getLogger("geonode.geoapps.views") + +_PERMISSION_MSG_DELETE = _("You are not permitted to delete this app.") +_PERMISSION_MSG_GENERIC = _("You do not have permissions for this app.") +_PERMISSION_MSG_LOGIN = _("You must be logged in to save this app") +_PERMISSION_MSG_SAVE = _("You are not permitted to save or edit this app.") +_PERMISSION_MSG_METADATA = _("You are not allowed to modify this app's metadata.") +_PERMISSION_MSG_VIEW = _("You are not allowed to view this app.") +_PERMISSION_MSG_UNKNOWN = _("An unknown error has occured.") + + +def _resolve_geoapp(request, id, permission="base.change_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs): + """ + Resolve the GeoApp by the provided typename and check the optional permission. + """ + + return resolve_object(request, GeoApp, {"pk": id}, permission=permission, permission_msg=msg, **kwargs) + + +@login_required +def new_geoapp(request, template="apps/app_new.html"): + access_token = None + if request and request.user: + access_token = get_or_create_token(request.user) + if access_token and not access_token.is_expired(): + access_token = access_token.token + else: + access_token = None + + if request.method == "GET": + _ctx = { + "user": request.user, + "access_token": access_token, + } + return render(request, template, context=_ctx) + + return HttpResponseRedirect(hookset.geoapp_list_url()) + + +@xframe_options_sameorigin +def geoapp_edit(request, geoappid, template="apps/app_edit.html"): + """ + The view that returns the app composer opened to + the app with the given app ID. + """ + try: + geoapp_obj = _resolve_geoapp(request, geoappid, "base.view_resourcebase", _PERMISSION_MSG_VIEW) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not geoapp_obj: + raise Http404(_("Not found")) + + # Call this first in order to be sure "perms_list" is correct + permissions_json = _perms_info_json(geoapp_obj) + + perms_list = geoapp_obj.get_user_perms(request.user) + + group = None + if geoapp_obj.group: + try: + group = GroupProfile.objects.get(slug=geoapp_obj.group.name) + except GroupProfile.DoesNotExist: + group = None + + r = geoapp_obj + if request.method in ("POST", "PATCH", "PUT"): + r = resource_manager.update(geoapp_obj.uuid, instance=geoapp_obj, notify=True) + + resource_manager.set_permissions( + geoapp_obj.uuid, instance=geoapp_obj, permissions=ast.literal_eval(permissions_json) + ) + + resource_manager.set_thumbnail(geoapp_obj.uuid, instance=geoapp_obj, overwrite=False) + + access_token = None + if request and request.user: + access_token = get_or_create_token(request.user) + if access_token and not access_token.is_expired(): + access_token = access_token.token + else: + access_token = None + + _config = json.dumps(r.blob) + _ctx = { + "appId": geoappid, + "appType": geoapp_obj.resource_type, + "config": _config, + "user": request.user, + "access_token": access_token, + "resource": geoapp_obj, + "group": group, + "perms_list": perms_list, + "permissions_json": permissions_json, + "preview": getattr(settings, "GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore"), + } + + return render(request, template, context=_ctx) + + +def geoapp_metadata_detail(request, geoappid, template="apps/app_metadata_detail.html", custom_metadata=None): + try: + geoapp_obj = _resolve_geoapp(request, geoappid, "view_resourcebase", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not geoapp_obj: + raise Http404(_("Not found")) + + group = None + if geoapp_obj.group: + try: + group = GroupProfile.objects.get(slug=geoapp_obj.group.name) + except ObjectDoesNotExist: + group = None + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + register_event(request, EventType.EVENT_VIEW_METADATA, geoapp_obj) + + return render( + request, + template, + context={ + "resource": geoapp_obj, + "group": group, + "SITEURL": site_url, + "custom_metadata": custom_metadata, + }, + ) + + +@login_required +@check_keyword_write_perms +def geoapp_metadata( + request, + geoappid, + template="apps/app_metadata.html", + ajax=True, + panel_template="layouts/app_panels.html", + custom_metadata=None, +): + geoapp_obj = None + try: + geoapp_obj = _resolve_geoapp(request, geoappid, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not geoapp_obj: + raise Http404(_("Not found")) + + # Add metadata_author or poc if missing + geoapp_obj.add_missing_metadata_author_or_poc() + resource_type = geoapp_obj.resource_type + topic_category = geoapp_obj.category + current_keywords = [keyword.name for keyword in geoapp_obj.keywords.all()] + + topic_thesaurus = geoapp_obj.tkeywords.all() + + if request.method == "POST": + geoapp_form = GeoAppForm(request.POST, instance=geoapp_obj, prefix="resource", user=request.user) + category_form = CategoryForm( + request.POST, + prefix="category_choice_field", + initial=( + int(request.POST["category_choice_field"]) + if "category_choice_field" in request.POST and request.POST["category_choice_field"] + else None + ), + ) + + if hasattr(settings, "THESAURUS"): + tkeywords_form = TKeywordForm(request.POST) + else: + tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords") + + else: + geoapp_form = GeoAppForm(instance=geoapp_obj, prefix="resource", user=request.user) + geoapp_form.disable_keywords_widget_for_non_superuser(request.user) + category_form = CategoryForm( + prefix="category_choice_field", initial=topic_category.id if topic_category else None + ) + + # Create THESAURUS widgets + lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, "THESAURUS_DEFAULT_LANG") else "en" + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + warnings.warn( + "The settings for Thesaurus has been moved to Model, \ + this feature will be removed in next releases", + DeprecationWarning, + ) + dataset_tkeywords = geoapp_obj.tkeywords.all() + tkeywords_list = "" + if dataset_tkeywords and len(dataset_tkeywords) > 0: + tkeywords_ids = dataset_tkeywords.values_list("id", flat=True) + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + el = settings.THESAURUS + thesaurus_name = el["name"] + try: + t = Thesaurus.objects.get(identifier=thesaurus_name) + for tk in t.thesaurus.filter(pk__in=tkeywords_ids): + tkl = tk.keyword.filter(lang=lang) + if len(tkl) > 0: + tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True))) + tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids + except Exception: + tb = traceback.format_exc() + logger.error(tb) + tkeywords_form = TKeywordForm(instance=geoapp_obj) + else: + tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords") + # set initial values for thesaurus form + for tid in tkeywords_form.fields: + values = [] + values = [keyword.id for keyword in topic_thesaurus if int(tid) == keyword.thesaurus.id] + tkeywords_form.fields[tid].initial = values + + if request.method == "POST" and geoapp_form.is_valid() and category_form.is_valid() and tkeywords_form.is_valid(): + new_keywords = current_keywords if request.keyword_readonly else geoapp_form.cleaned_data.pop("keywords") + new_regions = geoapp_form.cleaned_data.pop("regions") + + new_category = None + if ( + category_form + and "category_choice_field" in category_form.cleaned_data + and category_form.cleaned_data["category_choice_field"] + ): + new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"])) + geoapp_form.cleaned_data.pop("ptype") + + geoapp_obj = geoapp_form.instance + # update contact roles + geoapp_obj.set_contact_roles_from_metadata_edit(geoapp_form) + + vals = dict(category=new_category) + + geoapp_form.cleaned_data.pop("metadata") + extra_metadata = geoapp_form.cleaned_data.pop("extra_metadata") + + geoapp_form.save_linked_resources() + geoapp_form.cleaned_data.pop("linked_resources") + + vals.update({"resource_type": resource_type, "sourcetype": SOURCE_TYPE_LOCAL}) + + register_event(request, EventType.EVENT_CHANGE_METADATA, geoapp_obj) + if not ajax: + return HttpResponseRedirect(hookset.geoapp_detail_url(geoapp_obj)) + + message = geoapp_obj.id + + try: + # Keywords from THESAURUS management + # Rewritten to work with updated autocomplete + if not tkeywords_form.is_valid(): + return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400)) + + thesaurus_setting = getattr(settings, "THESAURUS", None) + if thesaurus_setting: + tkeywords_data = tkeywords_form.cleaned_data["tkeywords"] + tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"]) + geoapp_obj.tkeywords.set(tkeywords_data) + elif Thesaurus.objects.all().exists(): + fields = tkeywords_form.cleaned_data + geoapp_obj.tkeywords.set(tkeywords_form.cleanx(fields)) + + except Exception: + tb = traceback.format_exc() + logger.error(tb) + + if "group" in geoapp_form.changed_data: + vals["group"] = geoapp_form.cleaned_data.get("group") + if any([x in geoapp_form.changed_data for x in ["is_approved", "is_published"]]): + vals["is_approved"] = geoapp_form.cleaned_data.get("is_approved", geoapp_obj.is_approved) + vals["is_published"] = geoapp_form.cleaned_data.get("is_published", geoapp_obj.is_published) + else: + vals.pop("is_approved", None) + vals.pop("is_published", None) + + resource_manager.update( + geoapp_obj.uuid, + instance=geoapp_obj, + keywords=new_keywords, + regions=new_regions, + notify=True, + vals=vals, + extra_metadata=json.loads(extra_metadata), + ) + + resource_manager.set_thumbnail(geoapp_obj.uuid, instance=geoapp_obj, overwrite=False) + + return HttpResponse(json.dumps({"message": message})) + elif request.method == "POST" and ( + not geoapp_form.is_valid() or not category_form.is_valid() or not tkeywords_form.is_valid() + ): + errors_list = { + **geoapp_form.errors.as_data(), + **category_form.errors.as_data(), + **tkeywords_form.errors.as_data(), + } + logger.error(f"GeoApp Metadata form is not valid: {errors_list}") + out = {"success": False, "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()]} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + # - POST Request Ends here - + + # define contact role forms + contact_role_forms_context = {} + for role in geoapp_obj.get_multivalue_role_property_names(): + geoapp_form.fields[role].initial = [p.username for p in geoapp_obj.__getattribute__(role)] + role_form = ProfileForm(prefix=role) + role_form.hidden = True + contact_role_forms_context[f"{role}_form"] = role_form + + metadata_author_groups = get_user_visible_groups(request.user) + + if not request.user.can_publish(geoapp_obj): + geoapp_form.fields["is_published"].widget.attrs.update({"disabled": "true"}) + if not request.user.can_approve(geoapp_obj): + geoapp_form.fields["is_approved"].widget.attrs.update({"disabled": "true"}) + + register_event(request, EventType.EVENT_VIEW_METADATA, geoapp_obj) + return render( + request, + template, + context={ + "resource": geoapp_obj, + "geoapp": geoapp_obj, + "panel_template": panel_template, + "custom_metadata": custom_metadata, + "geoapp_form": geoapp_form, + "category_form": category_form, + "tkeywords_form": tkeywords_form, + "metadata_author_groups": metadata_author_groups, + "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False), + "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + "UI_MANDATORY_FIELDS": list( + set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", [])) + | set(getattr(settings, "UI_REQUIRED_FIELDS", [])) + ), + **contact_role_forms_context, + "UI_ROLES_IN_TOGGLE_VIEW": geoapp_obj.get_ui_toggled_role_property_names(), + }, + ) + + +@login_required +def geoapp_metadata_advanced(request, geoappid): + return geoapp_metadata(request, geoappid, template="apps/app_metadata_advanced.html") diff --git a/geonode/geoserver/__init__.py b/geonode/geoserver/__init__.py new file mode 100644 index 0000000..b900f15 --- /dev/null +++ b/geonode/geoserver/__init__.py @@ -0,0 +1,20 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +BACKEND_PACKAGE = "geonode.geoserver" diff --git a/geonode/geoserver/apps.py b/geonode/geoserver/apps.py new file mode 100644 index 0000000..9ddf9e7 --- /dev/null +++ b/geonode/geoserver/apps.py @@ -0,0 +1,81 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +from django.conf import settings +from django.utils.translation import gettext_noop as _ +from geonode.notifications_helper import NotificationsAppConfigBase +from django.apps import AppConfig + + +logger = logging.getLogger(__name__) + + +def run_setup_hooks(*args, **kwargs): + from django.db.models import signals + from geonode.layers.models import Dataset + from geonode.maps.models import MapLayer + from geonode.geoserver.signals import geoserver_pre_delete + from geonode.geoserver.signals import geoserver_pre_save_maplayer + + signals.pre_delete.connect(geoserver_pre_delete, sender=Dataset) + signals.pre_save.connect(geoserver_pre_save_maplayer, sender=MapLayer) + + +def set_resource_links(*args, **kwargs): + from geonode.utils import set_resource_default_links + from geonode.catalogue.models import catalogue_post_save + from geonode.layers.models import Dataset + + if settings.UPDATE_RESOURCE_LINKS_AT_MIGRATE: + _all_datasets = Dataset.objects.all() + for index, layer in enumerate(_all_datasets, start=1): + _lyr_name = layer.name + message = f"[{index} / {len(_all_datasets)}] Updating Dataset [{_lyr_name}] ..." + logger.debug(message) + try: + set_resource_default_links(layer, layer) + catalogue_post_save(instance=layer, sender=layer.__class__) + except Exception: + logger.exception(f"[ERROR] Dataset [{_lyr_name}] couldn't be updated") + + +class GeoserverAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.geoserver" + NOTIFICATIONS = ( + ( + "dataset_uploaded", + _("Dataset Uploaded"), + _("A layer was uploaded"), + ), + ( + "dataset_rated", + _("Rating for Dataset"), + _("A rating was given to a layer"), + ), + ) + + def ready(self): + super().ready() + run_setup_hooks() + # Connect the post_migrate signal with the _set_resource_links + # method to update links for each resource + from django.db.models import signals + + signals.post_migrate.connect(set_resource_links, sender=self) diff --git a/geonode/geoserver/context_processors.py b/geonode/geoserver/context_processors.py new file mode 100644 index 0000000..572a024 --- /dev/null +++ b/geonode/geoserver/context_processors.py @@ -0,0 +1,40 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.conf import settings +from django.urls import reverse +from geonode.geoserver.helpers import ogc_server_settings + + +def geoserver_urls(request): + """Global values to pass to templates""" + defaults = dict( + GEOSERVER_LOCAL_URL=ogc_server_settings.LOCATION, + GEOSERVER_PUBLIC_LOCATION=ogc_server_settings.public_url, + GEOSERVER_BASE_URL=ogc_server_settings.public_url, + UPLOADER_URL=reverse("importer_upload"), + LAYER_ANCILLARY_FILES_UPLOAD_URL=reverse("importer_upload"), + EXECUTION_STATUS_ENDPOINT=reverse("executionrequest-list"), + MAPFISH_PRINT_ENABLED=getattr(ogc_server_settings, "MAPFISH_PRINT_ENABLED", False), + PRINT_NG_ENABLED=getattr(ogc_server_settings, "PRINT_NG_ENABLED", False), + GEONODE_SECURITY_ENABLED=getattr(ogc_server_settings, "GEONODE_SECURITY_ENABLED", False), + TIME_ENABLED=getattr(settings, "UPLOADER", dict()).get("OPTIONS", dict()).get("TIME_ENABLED", False), + MOSAIC_ENABLED=getattr(settings, "UPLOADER", dict()).get("OPTIONS", dict()).get("MOSAIC_ENABLED", False), + ) + return defaults diff --git a/geonode/geoserver/createlayer/__init__.py b/geonode/geoserver/createlayer/__init__.py new file mode 100644 index 0000000..f3b257d --- /dev/null +++ b/geonode/geoserver/createlayer/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/createlayer/apps.py b/geonode/geoserver/createlayer/apps.py new file mode 100644 index 0000000..a3b9434 --- /dev/null +++ b/geonode/geoserver/createlayer/apps.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + + +class GeoNodeGeoserverCreatelayerAppConfig(AppConfig): + name = "geonode.geoserver.createlayer" + verbose_name = "GeoNode Geoserver Createlayer" diff --git a/geonode/geoserver/createlayer/forms.py b/geonode/geoserver/createlayer/forms.py new file mode 100644 index 0000000..671ab66 --- /dev/null +++ b/geonode/geoserver/createlayer/forms.py @@ -0,0 +1,45 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django import forms +from django.utils.translation import gettext_lazy as _ + +GEOMETRY_TYPES = ( + ("Point", _("Points")), + ("LineString", _("Lines")), + ("Polygon", _("Polygons")), +) + + +class NewDatasetForm(forms.Form): + """ + A form to create an empty layer in PostGIS. + """ + + name = forms.CharField(label=_("Dataset name"), max_length=255) + title = forms.CharField(label=_("Dataset title"), max_length=255) + geometry_type = forms.ChoiceField(label=_("Geometry type"), choices=GEOMETRY_TYPES) + + permissions = forms.CharField( + widget=forms.HiddenInput(attrs={"name": "permissions", "id": "permissions"}), required=False + ) + + attributes = forms.CharField( + widget=forms.HiddenInput(attrs={"name": "attributes", "id": "attributes"}), required=False, empty_value="{}" + ) diff --git a/geonode/geoserver/createlayer/management/__init__.py b/geonode/geoserver/createlayer/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/geoserver/createlayer/management/commands/__init__.py b/geonode/geoserver/createlayer/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/geoserver/createlayer/management/commands/createvectorlayer.py b/geonode/geoserver/createlayer/management/commands/createvectorlayer.py new file mode 100644 index 0000000..8ab30e0 --- /dev/null +++ b/geonode/geoserver/createlayer/management/commands/createvectorlayer.py @@ -0,0 +1,58 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand + +from geonode.people.utils import get_valid_user +from geonode.geoserver.createlayer.utils import create_dataset + + +class Command(BaseCommand): + help = "Create an empty PostGIS vector layer in GeoNode." + + def add_arguments(self, parser): + # positional arguments + parser.add_argument('name', type=str) + # named (optional arguments) + parser.add_argument('--user', + help='Name of the user account which should own the created layer') + parser.add_argument('--geometry', + default='Point', + help=('Geometry type of the layer to be created. ' + 'Can be Point, LineString or Polygon. Default is Point') + ) + parser.add_argument('--attributes', + help=('A json representation of the attributes to create. ' + 'Example: ' + '{ "field_str": "string", "field_int": "integer", ' + '"field_date": "date", "field_float": "float"}') + ) + parser.add_argument('--title', + default='No title', + help='Title for the layer to be created.' + ) + + def handle(self, *args, **options): + name = options.get('name') + username = options.get('user') + user = get_valid_user(username) + title = options.get('title') + geometry_type = options.get('geometry') + attributes = options.get('attributes') + create_dataset(name, title, user, geometry_type, attributes) diff --git a/geonode/geoserver/createlayer/templates/createlayer/dataset_create.html b/geonode/geoserver/createlayer/templates/createlayer/dataset_create.html new file mode 100644 index 0000000..e4fece0 --- /dev/null +++ b/geonode/geoserver/createlayer/templates/createlayer/dataset_create.html @@ -0,0 +1,97 @@ +{% extends "geonode_base.html" %} +{% load i18n %} +{% load base_tags %} +{% load static %} +{% load bootstrap_tags %} +{% load client_lib_tags %} + +{% block title %} {% trans "Create Dataset" %} - {{ block.super }} {% endblock %} + +{% block body_class %}layer create{% endblock %} + + +{% block head %} + +{{ block.super }} +{% endblock %} + +{% block body_outer %} + +
      +

      {% trans "Create an empty dataset" %}

      +
      + +
      +
      + + {% if error %} +
      +

      {{ error }}

      +
      + {% endif %} + +
      +
      + {% csrf_token %} + {{ form|as_bootstrap }} + +
      + +
      + + + +
      +
      + +
      +
      +{% endblock %} + + +{% block extra_script %} + +{{ block.super }} + + + +{% endblock extra_script %} diff --git a/geonode/geoserver/createlayer/tests.py b/geonode/geoserver/createlayer/tests.py new file mode 100644 index 0000000..eb18bc4 --- /dev/null +++ b/geonode/geoserver/createlayer/tests.py @@ -0,0 +1,139 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os + +from django.conf import settings +from django.urls import reverse + +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode import geoserver +from geonode import GeoNodeException +from geonode.layers.models import Dataset +from geonode.decorators import on_ogc_backend +from geonode.geoserver.signals import gs_catalog + +from .utils import create_dataset + + +class CreateLayerCoreTest(GeoNodeBaseTestSupport): + """ + Test createlayer application. + """ + + fixtures = ["initial_data.json", "bobby"] + + def setUp(self): + super().setUp() + # createlayer must use postgis as a datastore + # set temporary settings to use a postgis datastore + # DATASTORE_URL = 'postgis://geonode:geonode@localhost:5432/datastore' + # postgis_db = dj_database_url.parse(DATASTORE_URL, conn_max_age=0) + # settings.DATABASES['datastore'] = postgis_db + # settings.OGC_SERVER['default']['DATASTORE'] = 'datastore' + + def tearDown(self): + super(GeoNodeBaseTestSupport, self).tearDown() + # move to original settings + # settings.OGC_SERVER['default']['DATASTORE'] = '' + # del settings.DATABASES['datastore'] + # TODO remove stuff from django and geoserver catalog + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_dataset_creation(self): + """ + Try creating a layer. + """ + internal_apps_tests = os.environ.get("TEST_RUN_INTERNAL_APPS", None) + if not internal_apps_tests: + internal_apps_tests = settings.internal_apps_tests + else: + dataset_name = "point_dataset" + dataset_title = "A layer for points" + print(settings.DATABASES["datastore"]) + print("#######") + print(settings.OGC_SERVER["default"]["DATASTORE"]) + + create_dataset(dataset_name, dataset_title, "bobby", "Point") + + cat = gs_catalog + + # Check the layer is in the Django database + layer = Dataset.objects.get(name=dataset_name) + + # check if it is in geoserver + gs_dataset = cat.get_layer(dataset_name) + try: + self.assertIsNotNone(gs_dataset) + self.assertEqual(gs_dataset.name, dataset_name) + + resource = gs_dataset.resource + # we must have only one attibute ('the_geom') + self.assertEqual(len(resource.attributes), 1) + + # check layer corrispondence between django and geoserver + self.assertEqual(resource.title, dataset_title) + self.assertEqual(resource.projection, layer.srid) + + # check if layer detail page is accessible with client + response = self.client.get(reverse("dataset_embed", args=(f"geonode:{dataset_name}",))) + self.assertEqual(response.status_code, 200) + finally: + cat.delete(gs_dataset) + + def test_dataset_creation_with_wrong_geometry_type(self): + """ + Try creating a layer with uncorrect geometry type. + """ + with self.assertRaises(GeoNodeException): + create_dataset("wrong_geom_dataset", "A layer with wrong geometry", "bobby", "wrong_geometry") + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_dataset_creation_with_attributes(self): + """ + Try creating a layer with attributes. + """ + internal_apps_tests = os.environ.get("TEST_RUN_INTERNAL_APPS", None) + if not internal_apps_tests: + internal_apps_tests = settings.internal_apps_tests + + else: + attributes = """ + { + "field_str": "string", + "field_int": "integer", + "field_date": "date", + "field_float": "float" + } + """ + + dataset_name = "attributes_dataset" + dataset_title = "A layer with attributes" + + create_dataset(dataset_name, dataset_title, "bobby", "Point", attributes) + + cat = gs_catalog + gs_dataset = cat.get_layer(dataset_name) + try: + resource = gs_dataset.resource + + # we must have one attibute for the geometry, and 4 other ones + self.assertEqual(len(resource.attributes), 5) + finally: + cat.delete(gs_dataset) diff --git a/geonode/geoserver/createlayer/urls.py b/geonode/geoserver/createlayer/urls.py new file mode 100644 index 0000000..5521de8 --- /dev/null +++ b/geonode/geoserver/createlayer/urls.py @@ -0,0 +1,23 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.urls import re_path +from . import views + +urlpatterns = [re_path(r"$", views.dataset_create, name="dataset_create")] diff --git a/geonode/geoserver/createlayer/utils.py b/geonode/geoserver/createlayer/utils.py new file mode 100644 index 0000000..6f22749 --- /dev/null +++ b/geonode/geoserver/createlayer/utils.py @@ -0,0 +1,226 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import json +import uuid +import logging +import requests + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.gis.geos import Polygon +from django.template.defaultfilters import slugify + +from geonode import GeoNodeException +from geonode.layers.models import Dataset +from geonode.layers.utils import get_valid_name +from geonode.resource.manager import resource_manager +from geonode.geoserver.helpers import gs_catalog, ogc_server_settings, create_geoserver_db_featurestore + + +logger = logging.getLogger(__name__) + +BBOX = [-180, -90, 180, 90] +DATA_QUALITY_MESSAGE = "Created with GeoNode" + + +def create_dataset(name, title, owner_name, geometry_type, attributes=None): + """ + Create an empty layer in GeoServer and register it in GeoNode. + """ + # first validate parameters + if geometry_type not in ("Point", "LineString", "Polygon"): + msg = "geometry must be Point, LineString or Polygon" + logger.error(msg) + raise GeoNodeException(msg) + name = get_valid_name(name) + # we can proceed + logger.debug("Creating the layer in GeoServer") + workspace, datastore = create_gs_dataset(name, title, geometry_type, attributes) + logger.debug("Creating the layer in GeoNode") + return create_gn_dataset(workspace, datastore, name, title, owner_name) + + +def create_gn_dataset(workspace, datastore, name, title, owner_name): + """ + Associate a layer in GeoNode for a given layer in GeoServer. + """ + owner = get_user_model().objects.get(username=owner_name) + + layer = resource_manager.create( + str(uuid.uuid4()), + resource_type=Dataset, + defaults=dict( + name=name, + workspace=workspace.name, + store=datastore.name, + subtype="vector", + alternate=f"{workspace.name}:{name}", + title=title, + owner=owner, + srid="EPSG:4326", + bbox_polygon=Polygon.from_bbox(BBOX), + ll_bbox_polygon=Polygon.from_bbox(BBOX), + data_quality_statement=DATA_QUALITY_MESSAGE, + ), + ) + + to_update = {} + if settings.ADMIN_MODERATE_UPLOADS: + to_update["is_approved"] = to_update["was_approved"] = False + if settings.RESOURCE_PUBLISHING: + to_update["is_published"] = to_update["was_published"] = False + + resource_manager.update(layer.uuid, instance=layer, vals=to_update) + resource_manager.set_thumbnail(None, instance=layer) + return layer + + +def get_attributes(geometry_type, json_attrs=None): + """ + Convert a json representation of attributes to a Python representation. + + parameters: + + json_attrs + { + "field_str": "string", + "field_int": "integer", + "field_date": "date", + "field_float": "float" + } + + geometry_type: a string which can be "Point", "LineString" or "Polygon" + + Output: + [ + ['the_geom', u'com.vividsolutions.jts.geom.Polygon', {'nillable': False}], + ['field_str', 'java.lang.String', {'nillable': True}], + ['field_int', 'java.lang.Integer', {'nillable': True}], + ['field_date', 'java.util.Date', {'nillable': True}], + ['field_float', 'java.lang.Float', {'nillable': True}] + ] + """ + + lattrs = [] + gattr = [] + gattr.append("the_geom") + gattr.append(f"com.vividsolutions.jts.geom.{geometry_type}") + gattr.append({"nillable": False}) + lattrs.append(gattr) + if json_attrs: + jattrs = json.loads(json_attrs) + for jattr in jattrs.items(): + lattr = [] + attr_name = slugify(jattr[0]) + attr_type = jattr[1].lower() + if len(attr_name) == 0: + msg = f"You must provide an attribute name for attribute of type {attr_type}" + logger.error(msg) + raise GeoNodeException(msg) + if attr_type not in ("float", "date", "string", "integer"): + msg = f"{attr_type} is not a valid type for attribute {attr_name}" + logger.error(msg) + raise GeoNodeException(msg) + if attr_type == "date": + attr_type = f"java.util.{(attr_type[:1].upper() + attr_type[1:])}" + else: + attr_type = f"java.lang.{(attr_type[:1].upper() + attr_type[1:])}" + lattr.append(attr_name) + lattr.append(attr_type) + lattr.append({"nillable": True}) + lattrs.append(lattr) + return lattrs + + +def get_or_create_datastore(cat, workspace=None, charset="UTF-8"): + """ + Get a PostGIS database store or create it in GeoServer if does not exist. + """ + dsname = ogc_server_settings.datastore_db["NAME"] + ds = create_geoserver_db_featurestore(store_name=dsname, workspace=workspace) + return ds + + +def create_gs_dataset(name, title, geometry_type, attributes=None): + """ + Create an empty PostGIS layer in GeoServer with a given name, title, + geometry_type and attributes. + """ + + native_name = name + cat = gs_catalog + + # get workspace and store + workspace = cat.get_default_workspace() + + # get (or create the datastore) + datastore = get_or_create_datastore(cat, workspace) + + # check if datastore is of PostGIS type + if datastore.type != "PostGIS": + msg = "To use the createlayer application you must use PostGIS" + logger.error(msg) + raise GeoNodeException(msg) + + # check if layer is existing + resources = datastore.get_resources() + for resource in resources: + if resource.name == name: + msg = f"There is already a layer named {name} in {workspace}" + logger.error(msg) + raise GeoNodeException(msg) + + attributes = get_attributes(geometry_type, attributes) + attributes_block = "" + for spec in attributes: + att_name, binding, opts = spec + nillable = opts.get("nillable", False) + attributes_block += ( + "" + f"{att_name}" + f"{binding}" + f"{nillable}" + "" + ) + attributes_block += "" + + # TODO implement others srs and not only EPSG:4326 + xml = ( + "" + f"{name}" + f"{native_name}" + f"{title}" + "EPSG:4326" + f"{BBOX[0]}{BBOX[2]}{BBOX[1]}{BBOX[3]}" + f"EPSG:4326" + f"{attributes_block}" + "" + ) + + url = f"{ogc_server_settings.rest}/workspaces/{workspace.name}/datastores/{datastore.name}/featuretypes" + headers = {"Content-Type": "application/xml"} + _user, _password = ogc_server_settings.credentials + req = requests.post(url, data=xml, headers=headers, auth=(_user, _password)) + if req.status_code != 201: + logger.error(f"Request status code was: {req.status_code}") + logger.error(f"Response was: {req.text}") + raise Exception(f"Dataset could not be created in GeoServer {req.text}") + + cat.reload() + return workspace, datastore diff --git a/geonode/geoserver/createlayer/views.py b/geonode/geoserver/createlayer/views.py new file mode 100644 index 0000000..0a16517 --- /dev/null +++ b/geonode/geoserver/createlayer/views.py @@ -0,0 +1,63 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json + +from django.contrib.auth.decorators import login_required +from django.shortcuts import render +from django.template.defaultfilters import slugify +from django.shortcuts import redirect + +from geonode.security.permissions import DEFAULT_PERMS_SPEC + +from .forms import NewDatasetForm +from .utils import create_dataset + + +@login_required +def dataset_create(request, template="createlayer/dataset_create.html"): + """ + Create an empty layer. + """ + error = None + if request.method == "POST": + form = NewDatasetForm(request.POST) + if form.is_valid(): + try: + name = form.cleaned_data["name"] + name = slugify(name.replace(".", "_")) + title = form.cleaned_data["title"] + geometry_type = form.cleaned_data["geometry_type"] + attributes = form.cleaned_data["attributes"] + permissions = DEFAULT_PERMS_SPEC + layer = create_dataset(name, title, request.user.username, geometry_type, attributes) + layer.set_permissions(json.loads(permissions), created=True) + return redirect(layer) + except Exception as e: + error = f"{e} ({type(e)})" + else: + form = NewDatasetForm() + + ctx = { + "form": form, + "is_dataset": True, + "error": error, + } + + return render(request, template, context=ctx) diff --git a/geonode/geoserver/geofence.py b/geonode/geoserver/geofence.py new file mode 100644 index 0000000..8198a2f --- /dev/null +++ b/geonode/geoserver/geofence.py @@ -0,0 +1,470 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import itertools +import logging +import requests +from requests.auth import HTTPBasicAuth +import traceback +import urllib + +logger = logging.getLogger(__name__) + + +class GeofenceException(Exception): + pass + + +class Rule: + """_summary_ + A GeoFence Rule. + Provides the object to be rendered as JSON. + + e.g.: + {"Rule": + { + "priority": 0, + "userName": "admin", + "service": "WMS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + } + + Returns: + _type_: Rule + """ + + ALLOW = "ALLOW" + DENY = "DENY" + LIMIT = "LIMIT" + + CM_MIXED = "MIXED" + + def __init__( + self, + access: (str, bool), + priority=None, + workspace=None, + layer=None, + user=None, + group=None, + service=None, + request=None, + subfield=None, + geo_limit=None, + catalog_mode=None, + ) -> None: + self.fields = {} + + # access may be either a boolean or ALLOW/DENY/LIMIT + if access is True: + access = Rule.ALLOW + elif access is False: + access = Rule.DENY + + for field, value in ( + ("priority", priority), + ("userName", user), + ("roleName", f"ROLE_{group.upper()}" if group is not None and group != "*" else group), + ("service", service), + ("request", request), + ("subfield", subfield), + ("workspace", workspace), + ("layer", layer), + ("access", access), + ): + if value is not None and value != "*": + self.fields[field] = value + + limits = {} + for field, value in ( + ("allowedArea", geo_limit), + ("catalogMode", catalog_mode), + ): + if value is not None: + limits[field] = value + + if limits: + self.fields["limits"] = limits + + def set_priority(self, pri: int): + self.fields["priority"] = pri + + def get_object(self) -> dict: + logger.debug(f"Creating Rule object: {self.fields}") + return {"Rule": self.fields} + + +class Batch: + """_summary_ + A GeoFence Batch. + It's a list of operations that will be executed transactionally inside GeoFence. + + e.g.: + { + "Batch": { + "operations": [ + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 0, + "userName": "admin", + "service": "WMS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 1, + "userName": "admin", + "service": "GWC", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 2, + "userName": "admin", + "service": "WFS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 3, + "userName": "admin", + "service": "WPS", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + }, + { + "@service": "rules", + "@type": "insert", + "Rule": { + "priority": 4, + "userName": "admin", + "workspace": "geonode", + "layer": "san_andres_y_providencia_administrative", + "access": "ALLOW" + } + } + ] + } + } + + Returns: + _type_: Batch + """ + + def __init__(self, log_name=None) -> None: + self.operations = [] + self.log_name = f'"{log_name}"' if log_name else "" + + def __str__(self) -> str: + return super().__str__() + + def add_delete_rule(self, rule_id: int): + self.operations.append({"@service": "rules", "@type": "delete", "@id": rule_id}) + + def add_insert_rule(self, rule: Rule): + operation = { + "@service": "rules", + "@type": "insert", + } + operation.update(rule.get_object()) + self.operations.append(operation) + + def length(self) -> int: + return len(self.operations) + + def get_object(self) -> dict: + return {"Batch": {"operations": self.operations}} + + +class AutoPriorityBatch(Batch): + """_summary_ + A Batch that handles the priority of the inserted rules. + The first rule will have the declared `start_rule_pri`, next Rules will have the priority incremented. + """ + + def __init__(self, start_rule_pri: int, log_name=None) -> None: + super().__init__(log_name) + self.pri = itertools.count(start_rule_pri) + + def add_insert_rule(self, rule: Rule): + rule.set_priority(self.pri.__next__()) + super().add_insert_rule(rule) + + +class GeoFenceClient: + """_summary_ + A GeoFence REST client allowing to interact with the embedded GeoFence API (which is slightly incompatible + with the original standalone GeoFence API.) + The class methods map on GeoFence operations. + Functionalities needing more than one call are implemented within GeoFenceUtils. + """ + + def __init__(self, baseurl: str, username: str, pw: str) -> None: + if not baseurl.endswith("/"): + baseurl += "/" + + self.baseurl = baseurl + self.username = username + self.pw = pw + self.timeout = 60 + + def set_timeout(self, timeout: int): + self.timeout = timeout + + def invalidate_cache(self): + r = requests.put(f"{self.baseurl}ruleCache/invalidate", auth=HTTPBasicAuth(self.username, self.pw)) + + if r.status_code != 200: + logger.debug("Could not invalidate cache") + raise GeofenceException("Could not invalidate cache") + + def get_rules( + self, + page: int = None, + entries: int = None, + workspace: str = None, + workspace_any: bool = None, + layer: str = None, + layer_any: bool = None, + ): + if (page is None and entries is not None) or (page is not None and entries is None): + raise GeofenceException(f"Bad page/entries combination {page}/{entries}") + + try: + """ + curl -X GET -u admin:geoserver \ + http://:/geoserver/rest/geofence/rules.json?page={page}&entries={entries} + """ + params = {} + + if entries: + params.update({"page": page, "entries": entries}) + + for param, value in ( + ("workspace", workspace), + ("workspaceAny", workspace_any), + ("layer", layer), + ("layerAny", layer_any), + ): + if value is not None: + params[param] = value + + url = f"{self.baseurl}rules.json?{urllib.parse.urlencode(params)}" + + r = requests.get( + url, + headers={"Content-type": "application/json"}, + auth=HTTPBasicAuth(self.username, self.pw), + timeout=self.timeout, + verify=False, + ) + + if r.status_code != 200: + logger.debug(f"Could not retrieve GeoFence Rules from {url} -- code:{r.status_code} - {r.text}") + raise GeofenceException(f"Could not retrieve GeoFence Rules: [{r.status_code}]") + + return r.json() + except Exception as e: + logger.debug("Error while retrieving GeoFence rules", exc_info=e) + raise GeofenceException(f"Error while retrieving GeoFence rules: {e}") + + def get_rules_count(self): + """Get the number of available GeoFence Rules""" + try: + """ + curl -X GET -u admin:geoserver \ + http://:/geoserver/rest/geofence/rules/count.json + """ + r = requests.get( + f"{self.baseurl}rules/count.json", + headers={"Content-type": "application/json"}, + auth=HTTPBasicAuth(self.username, self.pw), + timeout=self.timeout, + verify=False, + ) + + if r.status_code != 200: + logger.debug(f"Could not retrieve GeoFence Rules count: [{r.status_code}] - {r.text}") + raise GeofenceException(f"Could not retrieve GeoFence Rules count: [{r.status_code}]") + + response = r.json() + return response["count"] + + except Exception as e: + logger.debug("Error while retrieving GeoFence rules count", exc_info=e) + raise GeofenceException(f"Error while retrieving GeoFence rules count: {e}") + + def insert_rule(self, rule: Rule): + try: + """ + curl -X POST -u admin:geoserver -H "Content-Type: text/xml" -d \ + "geonode{layer}ALLOW" \ + http://:/geoserver/rest/geofence/rules + """ + r = requests.post( + f"{self.baseurl}rules", + json=rule.get_object(), + auth=HTTPBasicAuth(self.username, self.pw), + timeout=self.timeout, + verify=False, + ) + + if r.status_code not in (200, 201): + logger.debug(f"Could not insert rule: [{r.status_code}] - {r.text}") + raise GeofenceException(f"Could not insert rule: [{r.status_code}] - {r.text}") + + except Exception as e: + logger.debug("Error while inserting rule", exc_info=e) + raise GeofenceException(f"Error while inserting rule: {e}") + + def run_batch(self, batch: Batch, timeout: int = None) -> bool: + if batch.length() == 0: + logger.debug(f"Skipping batch execution {batch.log_name}") + return False + + logger.debug(f"Running batch {batch.log_name} with {batch.length()} operations") + try: + """ + curl -X GET -u admin:geoserver \ + http://:/geoserver/rest/geofence/rules/count.json + """ + r = requests.post( + f"{self.baseurl}batch/exec", + json=batch.get_object(), + auth=HTTPBasicAuth(self.username, self.pw), + timeout=timeout or self.timeout, + verify=False, + ) + + if r.status_code != 200: + logger.debug( + f"Error while running batch {batch.log_name}: [{r.status_code}] - {r.content}" + f"\n {batch.get_object()}" + ) + raise GeofenceException(f"Error while running batch {batch.log_name}: [{r.status_code}]") + + return True + + except Exception as e: + logger.info(f"Error while requesting batch exec {batch.log_name}") + logger.debug(f"Error while requesting batch exec {batch.log_name} --> {batch.get_object()}", exc_info=e) + raise GeofenceException(f"Error while requesting batch execution {batch.log_name}: {e}") + + +class GeoFenceUtils: + def __init__(self, client: GeoFenceClient): + self.geofence = client + + def delete_all_rules(self): + """purge all existing GeoFence Cache Rules""" + rules_objs = self.geofence.get_rules() + rules = rules_objs["rules"] + + batch = Batch("Purge All") + for rule in rules: + batch.add_delete_rule(rule["id"]) + + logger.debug(f"Going to remove all {len(rules)} rules in geofence") + self.geofence.run_batch(batch) + + def collect_delete_layer_rules(self, workspace_name: str, layer_name: str, batch: Batch = None) -> Batch: + """Collect delete operations in a Batch for all rules related to a layer""" + + try: + # Scan GeoFence Rules associated to the Dataset + gs_rules = self.geofence.get_rules( + workspace=workspace_name, workspace_any=False, layer=layer_name, layer_any=False + ) + + if not batch: + batch = Batch(f"Delete {workspace_name}:{layer_name}") + + cnt = 0 + if gs_rules and gs_rules["rules"]: + logger.debug( + f"Going to collect {len(gs_rules['rules'])} rules for layer '{workspace_name}:{layer_name}'" + ) + for r in gs_rules["rules"]: + if r["layer"] and r["layer"] == layer_name: + batch.add_delete_rule(r["id"]) + cnt += 1 + else: + logger.warning(f"Bad rule retrieved for dataset '{workspace_name or ''}:{layer_name}': {r}") + + logger.debug(f"Adding {cnt} rule deletion operations for '{workspace_name or ''}:{layer_name}") + return batch + + except Exception as e: + logger.error(f"Error collecting rules for {workspace_name}:{layer_name}", exc_info=e) + tb = traceback.format_exc() + logger.debug(tb) + + def delete_layer_rules(self, workspace_name: str, layer_name: str) -> bool: + """Delete all Rules related to a specific Layer""" + try: + batch = self.collect_delete_layer_rules(workspace_name, layer_name) + if not batch: + logger.error(f"Error removing rules for {workspace_name}:{layer_name}") + return False + + logger.debug(f"Going to remove {batch.length()} rules for layer {workspace_name}:{layer_name}") + return self.geofence.run_batch(batch) + + except Exception as e: + logger.error(f"Error removing rules for {workspace_name}:{layer_name}", exc_info=e) + tb = traceback.format_exc() + logger.debug(tb) + return False + + def get_first_available_priority(self): + """Get the highest Rules priority""" + try: + rules_count = self.geofence.get_rules_count() + rules_objs = self.geofence.get_rules(page=rules_count - 1, entries=1) + if len(rules_objs["rules"]) > 0: + highest_priority = rules_objs["rules"][0]["priority"] + else: + highest_priority = 0 + return int(highest_priority) + 1 + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + return -1 diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py new file mode 100644 index 0000000..250b593 --- /dev/null +++ b/geonode/geoserver/helpers.py @@ -0,0 +1,2311 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import re +import sys +import copy +import time +import uuid +import json +import errno +import typing +import logging +import datetime +import tempfile +import traceback +import dataclasses + +from shutil import copyfile +from itertools import cycle +from collections import defaultdict +from os.path import basename, splitext, isfile +from urllib.parse import urlparse, urlencode, urlsplit, urljoin +from bs4 import BeautifulSoup +import xml.etree.ElementTree as ET + +from django.conf import settings +from django.utils import timezone +from django.db import transaction +from django.utils.module_loading import import_string +from django.core.exceptions import ImproperlyConfigured +from django.template.loader import render_to_string +from django.utils.translation import gettext_lazy as _ + +from geoserver.catalog import Catalog, FailedRequestError +from geoserver.resource import FeatureType, Coverage +from geoserver.store import ( + CoverageStore, + DataStore, + datastore_from_index, + coveragestore_from_index, + wmsstore_from_index, +) +from geoserver.support import DimensionInfo +from geoserver.workspace import Workspace +from gsimporter import Client +from lxml import etree, objectify +from owslib.etree import etree as dlxml +from owslib.wcs import WebCoverageService + +from geonode import GeoNodeException +from geonode.base.models import Link +from geonode.base.models import ResourceBase +from geonode.security.views import _perms_info_json +from geonode.catalogue.models import catalogue_post_save +from geonode.layers.models import Dataset, Attribute, Style +from geonode.layers.enumerations import LAYER_ATTRIBUTE_NUMERIC_DATA_TYPES + +from geonode.utils import ( + OGC_Servers_Handler, + http_client, + get_legend_url, + is_monochromatic_image, + set_resource_default_links, +) + +from .geofence import GeoFenceClient, GeoFenceUtils + +logger = logging.getLogger(__name__) + +temp_style_name_regex = r"[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}_ms_.*" + +LAYER_SUBTYPES = { + "dataStore": "vector", + "coverageStore": "raster", + "remoteStore": "remote", + "vectorTimeSeries": "vector_time", +} + +WPS_ACCEPTABLE_FORMATS = [ + ("application/json", "vector"), + ("application/arcgrid", "raster"), + ("image/tiff", "raster"), + ("image/png", "raster"), + ("image/jpeg", "raster"), + ("application/wfs-collection-1.0", "vector"), + ("application/wfs-collection-1.1", "vector"), + ("application/zip", "vector"), + ("text/csv", "vector"), +] + +DEFAULT_STYLE_NAME = ["generic", "line", "point", "polygon", "raster"] + + +if not hasattr(settings, "OGC_SERVER"): + msg = ( + "Please configure OGC_SERVER when enabling geonode.geoserver." + " More info can be found at " + "http://docs.geonode.org/en/2.10.x/basic/settings/index.html#ogc-server" + ) + raise ImproperlyConfigured(msg) + + +def check_geoserver_is_up(): + """Verifies all geoserver is running, + this is needed to be able to upload. + """ + url = f"{ogc_server_settings.LOCATION}" + req, content = http_client.get(url, user=_user) + msg = f"Cannot connect to the GeoServer at {url}\nPlease make sure you have started it." + logger.debug(req) + assert req.status_code == 200, msg + + +def _add_sld_boilerplate(symbolizer): + """ + Wrap an XML snippet representing a single symbolizer in the appropriate + elements to make it a valid SLD which applies that symbolizer to all features, + including format strings to allow interpolating a "name" variable in. + """ + return ( + """ + + + %(name)s + + %(name)s + %(name)s + + +""" + + symbolizer + + """ + + + + + +""" + ) + + +_raster_template = """ + + 1.0 + +""" + +_polygon_template = """ + + + %(bg)s + + + %(fg)s + 0.7 + + +""" + +_line_template = """ + + + %(bg)s + 3 + + + + + + + + + %(fg)s + + +""" + +_point_template = """ + + + + %(mark)s + + %(bg)s + + + %(fg)s + + + 10 + + +""" + +_style_templates = dict( + raster=_add_sld_boilerplate(_raster_template), + polygon=_add_sld_boilerplate(_polygon_template), + line=_add_sld_boilerplate(_line_template), + point=_add_sld_boilerplate(_point_template), +) + +STYLES_VERSION = {"1.0.0": "sld10", "1.1.0": "sld11"} + + +def _extract_style_version_from_sld(sld): + """ + Assume: SLD as a byte + """ + root = objectify.fromstring(sld) + try: + return STYLES_VERSION[root.attrib["version"].strip()] + except Exception: + return STYLES_VERSION["1.0.0"] + + +def _style_name(resource): + return _punc.sub("_", f"{resource.store.workspace.name}:{resource.name}") + + +def extract_name_from_sld(gs_catalog, sld, sld_file=None): + try: + if sld: + if isfile(sld): + with open(sld, "rb") as sld_file: + sld = sld_file.read() + if isinstance(sld, str): + sld = sld.encode("utf-8") + dom = etree.XML(sld, parser=etree.XMLParser(resolve_entities=False)) + elif sld_file and isfile(sld_file): + with open(sld_file, "rb") as sld_file: + sld = sld_file.read() + if isinstance(sld, str): + sld = sld.encode("utf-8") + dom = dlxml.parse(sld) + except Exception: + logger.exception("The uploaded SLD file is not valid XML") + raise Exception("The uploaded SLD file is not valid XML") + + named_dataset = dom.findall("{http://www.opengis.net/sld}NamedLayer") + user_dataset = dom.findall("{http://www.opengis.net/sld}UserLayer") + + el = None + if named_dataset and len(named_dataset) > 0: + user_style = named_dataset[0].findall("{http://www.opengis.net/sld}UserStyle") + if user_style and len(user_style) > 0: + el = user_style[0].findall("{http://www.opengis.net/sld}Name") + if len(el) == 0: + el = user_style[0].findall("{http://www.opengis.net/se}Name") + + if len(el) == 0: + el = named_dataset[0].findall("{http://www.opengis.net/sld}Name") + if len(el) == 0: + el = named_dataset[0].findall("{http://www.opengis.net/se}Name") + + if not el or len(el) == 0: + if user_dataset and len(user_dataset) > 0: + user_style = user_dataset[0].findall("{http://www.opengis.net/sld}UserStyle") + if user_style and len(user_style) > 0: + el = user_style[0].findall("{http://www.opengis.net/sld}Name") + if len(el) == 0: + el = user_style[0].findall("{http://www.opengis.net/se}Name") + + if len(el) == 0: + el = user_dataset[0].findall("{http://www.opengis.net/sld}Name") + if len(el) == 0: + el = user_dataset[0].findall("{http://www.opengis.net/se}Name") + + if not el or len(el) == 0: + if sld_file: + return splitext(basename(sld_file))[0] + else: + raise Exception("Please provide a name, unable to extract one from the SLD.") + + return el[0].text + + +def get_sld_for(gs_catalog, layer): + name = None + gs_dataset = None + gs_style = None + + _default_style = None + _max_retries, _tries = getattr(ogc_server_settings, "MAX_RETRIES", 2), 0 + try: + gs_dataset = gs_catalog.get_layer(layer.name) + if gs_dataset.default_style: + gs_style = gs_dataset.default_style.sld_body + set_dataset_style(layer, layer.alternate, gs_style) + name = gs_dataset.default_style.name + _default_style = gs_dataset.default_style + except Exception as e: + logger.debug(e) + name = None + + while not name and _tries < _max_retries: + try: + gs_dataset = gs_catalog.get_layer(layer.name) + if gs_dataset: + if gs_dataset.default_style: + gs_style = gs_dataset.default_style.sld_body + set_dataset_style(layer, layer.alternate, gs_style) + name = gs_dataset.default_style.name + if name: + break + except Exception as e: + logger.exception(e) + name = None + _tries += 1 + time.sleep(3) + + if not _default_style: + _default_style = layer.default_style if layer else None + name = _default_style.name if _default_style else None + gs_style = _default_style.sld_body if _default_style else None + + if not name: + msg = """ + GeoServer didn't return a default style for this layer. + Consider increasing OGC_SERVER MAX_RETRIES value.'' + """ + raise GeoNodeException(msg) + + # Detect geometry type if it is a FeatureType + res = gs_dataset.resource if gs_dataset else None + if res and res.resource_type == "featureType": + res.fetch() + ft = res.store.get_resources(name=res.name) + ft.fetch() + for attr in ft.dom.find("attributes"): + attr_binding = attr.find("binding") + if "jts.geom" in attr_binding.text: + if "Polygon" in attr_binding.text: + name = "polygon" + elif "Line" in attr_binding.text: + name = "line" + else: + name = "point" + + # FIXME: When gsconfig.py exposes the default geometry type for vector + # layers we should use that rather than guessing based on the auto-detected + # style. + if name in _style_templates: + fg, bg, mark = next(_style_contexts) + return _style_templates[name] % dict(name=layer.name, fg=fg, bg=bg, mark=mark) + else: + return gs_style + + +def set_dataset_style(saved_dataset, title, sld, base_file=None): + # Check SLD is valid + try: + if sld: + if isfile(sld): + with open(sld, "rb") as sld_file: + sld = sld_file.read() + + elif isinstance(sld, str): + sld = sld.strip("b'\n") + sld = re.sub(r"(\\r)|(\\n)", "", sld).encode("UTF-8") + etree.XML(sld, parser=etree.XMLParser(resolve_entities=False)) + elif base_file and isfile(base_file): + with open(base_file, "rb") as sld_file: + sld = sld_file.read() + dlxml.parse(base_file) + except Exception: + logger.exception("The uploaded SLD file is not valid XML") + raise Exception("The uploaded SLD file is not valid XML") + + # Check Dataset's available styles + match = None + styles = list(saved_dataset.styles.all()) + [saved_dataset.default_style] + for style in styles: + if style and style.name == saved_dataset.name: + match = style + break + layer = gs_catalog.get_layer(title) + style = None + if match is None: + try: + style = gs_catalog.get_style(saved_dataset.name, workspace=saved_dataset.workspace) or gs_catalog.get_style( + saved_dataset.name + ) + if not style: + style = gs_catalog.create_style( + saved_dataset.name, sld, overwrite=False, raw=True, workspace=saved_dataset.workspace + ) + except Exception as e: + logger.exception(e) + else: + try: + _sld_format = _extract_style_version_from_sld(sld) + style = gs_catalog.create_style( + saved_dataset.name, + sld, + overwrite=True, + raw=True, + style_format=_sld_format, + workspace=saved_dataset.workspace, + ) + except Exception as e: + logger.exception(e) + + if layer and style: + _old_styles = [] + _old_styles.append(gs_catalog.get_style(name=saved_dataset.name)) + _old_styles.append(gs_catalog.get_style(name=f"{saved_dataset.workspace}_{saved_dataset.name}")) + if layer.default_style and layer.default_style.name: + _old_styles.append(gs_catalog.get_style(name=layer.default_style.name)) + _old_styles.append( + gs_catalog.get_style(name=layer.default_style.name, workspace=layer.default_style.workspace) + ) + layer.default_style = style + gs_catalog.save(layer) + for _s in _old_styles: + try: + gs_catalog.delete(_s) + Link.objects.filter( + resource=saved_dataset.resourcebase_ptr, name="Legend", url__contains=f"STYLE={_s.name}" + ).delete() + except Exception as e: + logger.debug(e) + set_styles(saved_dataset, gs_catalog) + + +def cascading_delete(dataset_name=None, catalog=None): + if not dataset_name: + return + cat = catalog or gs_catalog + resource = None + workspace = None + try: + if dataset_name.find(":") != -1 and len(dataset_name.split(":")) == 2: + workspace, name = dataset_name.split(":") + ws = cat.get_workspace(workspace) + store = None + try: + store = get_store(cat, name, workspace=ws) + except FailedRequestError: + if ogc_server_settings.DATASTORE: + try: + layers = Dataset.objects.filter(alternate=dataset_name) + for layer in layers: + store = get_store(cat, layer.store, workspace=ws) + except FailedRequestError: + logger.debug("the store was not found in geoserver") + else: + logger.debug("the store was not found in geoserver") + if ws is None or store is None: + logger.debug("cascading delete was called on a layer where the workspace was not found") + resource = cat.get_resource(name=name, store=store, workspace=workspace) + else: + resource = cat.get_resource(name=dataset_name) + except OSError as e: + if e.errno == errno.ECONNREFUSED: + msg = ( + f'Could not connect to geoserver at "{ogc_server_settings.LOCATION}"' + f'to save information for layer "{dataset_name}"' + ) + logger.error(msg) + return None + else: + raise e + finally: + # Let's reset the connections first + cat._cache.clear() + cat.reset() + cat.reload() + + if resource is None: + # If there is no associated resource, + # this method can not delete anything. + # Let's return and make a note in the log. + logger.debug("cascading_delete was called with a non existent resource") + return + resource_name = resource.name + lyr = None + try: + lyr = cat.get_layer(resource_name) + except Exception as e: + logger.debug(e) + if lyr is not None: # Already deleted + store = resource.store + styles = lyr.styles + try: + styles = styles + [lyr.default_style] + except Exception: + pass + if workspace: + gs_styles = [x for x in cat.get_styles(names=[f"{workspace}_{resource_name}"])] + styles = styles + gs_styles + if settings.DEFAULT_WORKSPACE and settings.DEFAULT_WORKSPACE != workspace: + gs_styles = [x for x in cat.get_styles(names=[f"{settings.DEFAULT_WORKSPACE}_{resource_name}"])] + styles = styles + gs_styles + cat.delete(lyr) + for s in styles: + if s is not None and s.name not in _default_style_names: + try: + logger.debug(f"Trying to delete Style [{s.name}]") + cat.delete(s, purge="true") + except Exception as e: + # Trying to delete a shared style will fail + # We'll catch the exception and log it. + logger.debug(e) + + # Due to a possible bug of geoserver, we need this trick for now + # TODO: inspect the issue reported by this hack. Should be solved + # with GS 2.7+ + try: + cat.delete(resource, recurse=True) # This may fail + except Exception: + pass + + if ( + store.resource_type == "dataStore" + and "dbtype" in store.connection_parameters + and store.connection_parameters["dbtype"] == "postgis" + ): + delete_from_postgis(resource_name, store) + else: + # AF: for the time being this one mitigates the issue #8671 + # until we find a suitable solution for the GeoTools ImageMosaic plugin + # ref: https://github.com/geotools/geotools/blob/main/modules/plugin/imagemosaic/src/main/java/org/geotools/gce/imagemosaic/catalog/AbstractGTDataStoreGranuleCatalog.java#L753 + if store.resource_type == "coverageStore" and store.type != "ImageMosaic": + try: + logger.debug(f" - Going to purge the {store.resource_type} : {store.href}") + cat.reset() # this resets the coverage readers and unlocks the files + cat.delete(store, purge="all", recurse=True) + # cat.reload() # this preservers the integrity of geoserver + except Exception as e: + # Trying to recursively purge a store may fail + # We'll catch the exception and log it. + logger.debug(e) + else: + try: + if not store.get_resources(): + cat.delete(store, recurse=True) + except Exception as e: + # Catch the exception and log it. + logger.debug(e) + + +def delete_from_postgis(dataset_name, store): + """ + Delete a table from PostGIS (because Geoserver won't do it yet); + to be used after deleting a layer from the system. + """ + import psycopg2 + + # we will assume that store/database may change (when using shard for example) + # but user and password are the ones from settings (DATASTORE_URL) + db = ogc_server_settings.datastore_db + db_name = store.connection_parameters["database"] + user = db["USER"] + password = db["PASSWORD"] + host = store.connection_parameters["host"] + port = store.connection_parameters["port"] + conn = None + try: + conn = psycopg2.connect(dbname=db_name, user=user, host=host, port=port, password=password) + cur = conn.cursor() + cur.execute(f"SELECT DropGeometryTable ('{dataset_name}')") + conn.commit() + except Exception as e: + logger.error("Error deleting PostGIS table %s:%s", dataset_name, str(e)) + finally: + try: + if conn: + conn.close() + except Exception as e: + logger.error("Error closing PostGIS conn %s:%s", dataset_name, str(e)) + + +def gs_slurp( + ignore_errors=False, + verbosity=1, + console=None, + owner=None, + workspace=None, + store=None, + filter=None, + skip_unadvertised=False, + skip_geonode_registered=False, + remove_deleted=False, + permissions=None, + execute_signals=False, +): + """Configure the layers available in GeoServer in GeoNode. + It returns a list of dictionaries with the name of the layer, + the result of the operation and the errors and traceback if it failed. + """ + from geonode.resource.manager import resource_manager + + if console is None: + console = open(os.devnull, "w") + + if verbosity > 0: + print("Inspecting the available layers in GeoServer ...", file=console) + + cat = gs_catalog + + if workspace is not None and workspace: + workspace = cat.get_workspace(workspace) + if workspace is None: + resources = [] + else: + # obtain the store from within the workspace. if it exists, obtain resources + # directly from store, otherwise return an empty list: + if store is not None: + store = get_store(cat, store, workspace=workspace) + if store is None: + resources = [] + else: + resources = cat.get_resources(stores=[store]) + else: + resources = cat.get_resources(workspaces=[workspace]) + elif store is not None: + store = get_store(cat, store) + resources = cat.get_resources(stores=[store]) + else: + resources = cat.get_resources() + + if remove_deleted: + resources_for_delete_compare = resources[:] + workspace_for_delete_compare = workspace + # filter out layers for delete comparison with GeoNode layers by following criteria: + # enabled = true, if --skip-unadvertised: advertised = true, but + # disregard the filter parameter in the case of deleting layers + try: + resources_for_delete_compare = [k for k in resources_for_delete_compare if k.enabled in {"true", True}] + if skip_unadvertised: + resources_for_delete_compare = [ + k for k in resources_for_delete_compare if k.advertised in {"true", True} + ] + except Exception: + if ignore_errors: + pass + else: + raise + + if filter: + resources = [k for k in resources if filter in k.name] + + # filter out layers depending on enabled, advertised status: + _resources = [] + for k in resources: + try: + if k.enabled in {"true", True}: + _resources.append(k) + except Exception: + if ignore_errors: + continue + else: + raise + # resources = [k for k in resources if k.enabled in {"true", True}] + resources = _resources + if skip_unadvertised: + try: + resources = [k for k in resources if k.advertised in {"true", True}] + except Exception: + if ignore_errors: + pass + else: + raise + + # filter out layers already registered in geonode + dataset_names = Dataset.objects.values_list("alternate", flat=True) + if skip_geonode_registered: + try: + resources = [k for k in resources if f"{k.workspace.name}:{k.name}" not in dataset_names] + except Exception: + if ignore_errors: + pass + else: + raise + + # TODO: Should we do something with these? + # i.e. look for matching layers in GeoNode and also disable? + # disabled_resources = [k for k in resources if k.enabled == "false"] + + number = len(resources) + if verbosity > 0: + msg = "Found %d layers, starting processing" % number + print(msg, file=console) + output = { + "stats": { + "failed": 0, + "updated": 0, + "created": 0, + "deleted": 0, + }, + "layers": [], + "deleted_datasets": [], + } + start = datetime.datetime.now(timezone.get_current_timezone()) + for i, resource in enumerate(resources): + name = resource.name + the_store = resource.store + workspace = the_store.workspace + layer = None + try: + created = False + layer = Dataset.objects.filter(name=name, workspace=workspace.name).first() + if not layer: + layer = resource_manager.create( + str(uuid.uuid4()), + resource_type=Dataset, + defaults=dict( + name=name, + workspace=workspace.name, + store=the_store.name, + subtype=get_dataset_storetype(the_store.resource_type), + alternate=f"{workspace.name}:{resource.name}", + title=resource.title or _("No title provided"), + abstract=resource.abstract or _("No abstract provided"), + owner=owner, + ), + ) + created = True + # Hide the resource until finished + layer.set_processing_state("RUNNING") + bbox = resource.native_bbox + ll_bbox = resource.latlon_bbox + try: + layer.set_bbox_polygon([bbox[0], bbox[2], bbox[1], bbox[3]], resource.projection) + except GeoNodeException as e: + if not ll_bbox: + raise + else: + logger.exception(e) + layer.srid = "EPSG:4326" + layer.set_ll_bbox_polygon([ll_bbox[0], ll_bbox[2], ll_bbox[1], ll_bbox[3]]) + + # sync permissions in GeoFence + perm_spec = json.loads(_perms_info_json(layer)) + resource_manager.set_permissions(layer.uuid, permissions=perm_spec) + + # recalculate the layer statistics + set_attributes_from_geoserver(layer, overwrite=True) + + # in some cases we need to explicitily save the resource to execute the signals + # (for sure when running updatelayers) + resource_manager.update(layer.uuid, instance=layer, notify=execute_signals) + + # Creating the Thumbnail + resource_manager.set_thumbnail(layer.uuid, overwrite=True, check_bbox=False) + + except Exception as e: + # Hide the resource until finished + if layer: + layer.set_processing_state("FAILED") + if ignore_errors: + status = "failed" + exception_type, error, traceback = sys.exc_info() + else: + if verbosity > 0: + msg = "Stopping process because --ignore-errors was not set and an error was found." + print(msg, file=sys.stderr) + raise Exception(f"Failed to process {resource.name}") from e + if layer is None: + if ignore_errors: + status = "failed" + exception_type, error, traceback = sys.exc_info() + else: + if verbosity > 0: + msg = "Stopping process because --ignore-errors was not set and an error was found." + print(msg, file=sys.stderr) + raise Exception(f"Failed to process {resource.name}") + else: + if created: + if not permissions: + layer.set_default_permissions() + else: + layer.set_permissions(permissions) + + status = "created" + output["stats"]["created"] += 1 + else: + status = "updated" + output["stats"]["updated"] += 1 + + msg = f"[{status}] Dataset {name} ({(i + 1)}/{number})" + info = {"name": name, "status": status} + if status == "failed": + output["stats"]["failed"] += 1 + info["traceback"] = traceback + info["exception_type"] = exception_type + info["error"] = error + output["layers"].append(info) + if verbosity > 0: + print(msg, file=console) + + if remove_deleted: + q = Dataset.objects.filter() + if workspace_for_delete_compare is not None: + if isinstance(workspace_for_delete_compare, Workspace): + q = q.filter(workspace__exact=workspace_for_delete_compare.name) + else: + q = q.filter(workspace__exact=workspace_for_delete_compare) + if store is not None: + if isinstance(store, CoverageStore) or isinstance(store, DataStore): + q = q.filter(store__exact=store.name) + else: + q = q.filter(store__exact=store) + logger.debug("Executing 'remove_deleted' logic") + logger.debug("GeoNode Layers Found:") + + # compare the list of GeoNode layers obtained via query/filter with valid resources found in GeoServer + # filtered per options passed to updatelayers: --workspace, --store, --skip-unadvertised + # add any layers not found in GeoServer to deleted_datasets (must match + # workspace and store as well): + deleted_datasets = [] + for layer in q: + logger.debug( + "GeoNode Dataset info: name: %s, workspace: %s, store: %s", layer.name, layer.workspace, layer.store + ) + dataset_found_in_geoserver = False + for resource in resources_for_delete_compare: + # if layer.name matches a GeoServer resource, check also that + # workspace and store match, mark valid: + if layer.name == resource.name: + if layer.workspace == resource.workspace.name and layer.store == resource.store.name: + logger.debug( + "Matches GeoServer layer: name: %s, workspace: %s, store: %s", + resource.name, + resource.workspace.name, + resource.store.name, + ) + dataset_found_in_geoserver = True + if not dataset_found_in_geoserver: + logger.debug("----- Dataset %s not matched, marked for deletion ---------------", layer.name) + deleted_datasets.append(layer) + + number_deleted = len(deleted_datasets) + if verbosity > 0: + msg = ( + "\nFound %d layers to delete, starting processing" % number_deleted + if number_deleted > 0 + else "\nFound %d layers to delete" % number_deleted + ) + print(msg, file=console) + + for i, layer in enumerate(deleted_datasets): + logger.debug( + "GeoNode Dataset to delete: name: %s, workspace: %s, store: %s", + layer.name, + layer.workspace, + layer.store, + ) + try: + # delete ratings, and taggit tags: + layer.keywords.clear() + + layer.delete() + output["stats"]["deleted"] += 1 + status = "delete_succeeded" + except Exception: + status = "delete_failed" + + msg = f"[{status}] Dataset {layer.name} ({(i + 1)}/{number_deleted})" + info = {"name": layer.name, "status": status} + if status == "delete_failed": + exception_type, error, traceback = sys.exc_info() + info["traceback"] = traceback + info["exception_type"] = exception_type + info["error"] = error + output["deleted_datasets"].append(info) + if verbosity > 0: + print(msg, file=console) + + finish = datetime.datetime.now(timezone.get_current_timezone()) + td = finish - start + output["stats"]["duration_sec"] = td.microseconds / 1000000 + td.seconds + td.days * 24 * 3600 + return output + + +def get_stores(store_type=None): + cat = gs_catalog + stores = cat.get_stores() + store_list = [] + for store in stores: + store.fetch() + stype = store.dom.find("type").text.lower() + if store_type and store_type.lower() == stype: + store_list.append({"name": store.name, "type": stype}) + elif store_type is None: + store_list.append({"name": store.name, "type": stype}) + return store_list + + +def set_attributes(layer, attribute_map, overwrite=False, attribute_stats=None): + """*layer*: a geonode.layers.models.Dataset instance + *attribute_map*: a list of 2-lists specifying attribute names and types, + example: [ ['id', 'Integer'], ... ] + *overwrite*: replace existing attributes with new values if name/type matches. + *attribute_stats*: dictionary of return values from get_attribute_statistics(), + of the form to get values by referencing attribute_stats[][]. + """ + # we need 3 more items; description, attribute_label, and display_order + attribute_map_dict = { + "field": 0, + "ftype": 1, + "description": 2, + "label": 3, + "display_order": 4, + } + for attribute in attribute_map: + if len(attribute) == 2: + attribute.extend((None, None, 0)) + + attributes = layer.attribute_set.all() + # Delete existing attributes if they no longer exist in an updated layer + for la in attributes: + lafound = False + for attribute in attribute_map: + field, ftype, description, label, display_order = attribute + if field == la.attribute: + lafound = True + # store description and attribute_label in attribute_map + attribute[attribute_map_dict["description"]] = la.description + attribute[attribute_map_dict["label"]] = la.attribute_label + attribute[attribute_map_dict["display_order"]] = la.display_order + if overwrite or not lafound: + logger.debug("Going to delete [%s] for [%s]", la.attribute, layer.name) + la.delete() + + # Add new layer attributes if they doesn't exist already + if attribute_map: + iter = len(Attribute.objects.filter(dataset=layer)) + 1 + for attribute in attribute_map: + field, ftype, description, label, display_order = attribute + if field: + _gs_attrs = Attribute.objects.filter(dataset=layer, attribute=field) + if _gs_attrs.count() == 1: + la = _gs_attrs.get() + else: + if _gs_attrs.exists(): + _gs_attrs.delete() + la = Attribute.objects.create(dataset=layer, attribute=field) + la.visible = ftype.find("gml:") != 0 + la.attribute_type = ftype + la.description = description + la.attribute_label = label + la.display_order = iter + iter += 1 + if not attribute_stats or layer.name not in attribute_stats or field not in attribute_stats[layer.name]: + result = None + else: + result = attribute_stats[layer.name][field] + if result: + logger.debug("Generating layer attribute statistics") + la.count = result["Count"] + la.min = result["Min"] + la.max = result["Max"] + la.average = result["Average"] + la.median = result["Median"] + la.stddev = result["StandardDeviation"] + la.sum = result["Sum"] + la.unique_values = result["unique_values"] + la.last_stats_updated = datetime.datetime.now(timezone.get_current_timezone()) + try: + la.save() + except Exception as e: + logger.exception(e) + else: + logger.debug("No attributes found") + + +def set_attributes_from_geoserver(layer, overwrite=False): + """ + Retrieve layer attribute names & types from Geoserver, + then store in GeoNode database using Attribute model + """ + attribute_map = [] + if getattr(layer, "remote_service") and layer.remote_service: + server_url = layer.remote_service.service_url + if layer.remote_service.operations.get("GetCapabilities", None) and layer.remote_service.operations.get( + "GetCapabilities" + ).get("methods"): + for _method in layer.remote_service.operations.get("GetCapabilities").get("methods"): + if _method.get("type", "").upper() == "GET": + server_url = _method.get("url", server_url) + break + else: + server_url = ogc_server_settings.LOCATION + if layer.subtype in ["tileStore", "remote"] and layer.remote_service.ptype == "gxp_arcrestsource": + logger.info(f"Getting info for {layer.subtype} '{layer.alternate or layer.typename}'") + dft_url = f"{server_url}{(layer.alternate or layer.typename)}?f=json" + try: + # The code below will fail if http_client cannot be imported + req, body = http_client.get(dft_url, user=_user) + body = json.loads(body) + attribute_map = [ + [n["name"], _esri_types[n["type"]]] for n in body["fields"] if n.get("name") and n.get("type") + ] + except Exception: + logger.warning( + f"Error while retrieving info for {layer.subtype} '{layer.alternate or layer.typename}'", exc_info=True + ) + attribute_map = [] + elif layer.subtype in {"vector", "tileStore", "remote", "wmsStore", "vector_time"}: + typename = layer.alternate if layer.alternate else layer.typename + logger.info(f"Getting WFS info for {layer.subtype} '{typename}'") + dft_url_path = re.sub(r"\/wms\/?$", "/", server_url) + dft_query = urlencode( + {"service": "wfs", "version": "1.0.0", "request": "DescribeFeatureType", "typename": typename} + ) + dft_url = urljoin(dft_url_path, f"ows?{dft_query}") + logger.debug(f"WFS URL is {dft_url}") + try: + # The code below will fail if http_client cannot be imported or WFS not supported + doc = _get_xml(dft_url) + xsd = "{http://www.w3.org/2001/XMLSchema}" + path = f".//{xsd}extension/{xsd}sequence/{xsd}element" + attribute_map = [ + [n.attrib["name"], n.attrib["type"]] + for n in doc.findall(path) + if n.attrib.get("name") and n.attrib.get("type") + ] + except Exception: + logger.warning( + f"Error while retrieving WFS info for {layer.subtype} '{typename}'... will try WMS", exc_info=True + ) + attribute_map = [] + # Try WMS instead + dft_url = ( + server_url + + "?" + + urlencode( + { + "service": "wms", + "version": "1.0.0", + "request": "GetFeatureInfo", + "bbox": ",".join([str(x) for x in layer.bbox]), + "LAYERS": layer.alternate, + "QUERY_LAYERS": typename, + "feature_count": 1, + "width": 1, + "height": 1, + "srs": "EPSG:4326", + "info_format": "text/html", + "x": 1, + "y": 1, + } + ) + ) + try: + logger.info(f"Getting WMS info for {layer.subtype} '{layer.alternate or layer.typename}'") + logger.debug(f"WMS URL is {dft_url}") + body = _get_from_catalog(dft_url) + soup = BeautifulSoup(body, features="lxml") + for field in soup.findAll("th"): + if field.string is None: + field_name = field.contents[0].string + else: + field_name = field.string + attribute_map.append([field_name, "xsd:string"]) + except Exception: + logger.warning(f"Error while retrieving WMS info for {layer.subtype} '{typename}'", exc_info=True) + attribute_map = [] + elif layer.subtype in ["raster"]: + typename = layer.alternate if layer.alternate else layer.typename + logger.info(f"Getting WCS info for {layer.subtype} '{typename}'") + dc_url = f"{server_url}wcs?{urlencode({'service': 'wcs', 'version': '1.1.0', 'request': 'DescribeCoverage', 'identifiers': typename})}" + try: + doc = _get_xml(dc_url) + wcs = "{http://www.opengis.net/wcs/1.1.1}" + path = f".//{wcs}Axis/{wcs}AvailableKeys/{wcs}Key" + attribute_map = [[n.text, "raster"] for n in doc.findall(path)] + except Exception: + logger.warning(f"Error while retrieving WCS info for {layer.subtype} '{typename}'", exc_info=True) + attribute_map = [] + # Get attribute statistics & package for call to really_set_attributes() + attribute_stats = defaultdict(dict) + # Add new layer attributes if they don't already exist + for attribute in attribute_map: + field, ftype = attribute + if field is not None: + if Attribute.objects.filter(dataset=layer, attribute=field).exists(): + continue + elif is_dataset_attribute_aggregable(layer.subtype, field, ftype): + logger.debug("Generating layer attribute statistics") + result = get_attribute_statistics(layer.alternate or layer.typename, field) + else: + result = None + attribute_stats[layer.name][field] = result + logger.info(f"Found {len(attribute_map)} attributes for {layer.subtype}") + set_attributes(layer, attribute_map, overwrite=overwrite, attribute_stats=attribute_stats) + + +def get_dataset(layer, gs_catalog: Catalog): + gs_catalog.reset() + gs_dataset = None + try: + gs_dataset = gs_catalog.get_layer(layer.name) + except Exception: + tb = traceback.format_exc() + logger.exception(tb) + + if not gs_dataset: + try: + gs_dataset = gs_catalog.get_layer(layer.alternate or layer.typename) + except Exception: + tb = traceback.format_exc() + logger.error(tb) + logger.exception("No GeoServer Dataset found!") + return gs_dataset + + +def clean_styles(layer, gs_catalog: Catalog): + try: + # Cleanup Styles without a Workspace + style = None + gs_catalog.reset() + gs_dataset = get_dataset(layer, gs_catalog) + if gs_dataset is not None: + logger.debug(f'clean_styles: Retrieving style "{gs_dataset.default_style.name}" for cleanup') + style = gs_catalog.get_style(name=gs_dataset.default_style.name, workspace=None, recursive=True) + if style: + gs_catalog.delete(style, purge=True, recurse=False) + logger.debug(f"clean_styles: Style removed: {gs_dataset.default_style.name}") + else: + logger.debug(f"clean_styles: Style does not exist: {gs_dataset.default_style.name}") + except Exception as e: + logger.warning(f"Could not clean style for layer {layer.name}", exc_info=e) + logger.debug(f"Could not clean style for layer {layer.name} - STACK INFO", stack_info=True) + + +def set_styles(layer, gs_catalog: Catalog): + style_set = [] + gs_dataset = get_dataset(layer, gs_catalog) + if gs_dataset: + default_style = gs_dataset.get_full_default_style() + if default_style: + # make sure we are not using a default SLD (which won't be editable) + layer.default_style, _gs_default_style = save_style(default_style, layer) + try: + if ( + default_style.name != _gs_default_style.name + or default_style.workspace != _gs_default_style.workspace + ): + logger.debug(f'set_style: Setting default style "{_gs_default_style.name}" for layer "{layer.name}') + + gs_dataset.default_style = _gs_default_style + gs_catalog.save(gs_dataset) + if default_style.name not in DEFAULT_STYLE_NAME: + logger.debug( + f'set_style: Retrieving no-workspace default style "{default_style.name}" for deletion' + ) + style_to_delete = gs_catalog.get_style(name=default_style.name, workspace=None, recursive=True) + if style_to_delete: + gs_catalog.delete(style_to_delete, purge=True, recurse=False) + logger.debug(f"set_style: No-ws default style deleted: {default_style.name}") + else: + logger.debug(f"set_style: No-ws default style does not exist: {default_style.name}") + except Exception as e: + logger.error( + f'Error setting default style "{_gs_default_style.name}" for layer "{layer.name}', exc_info=e + ) + + style_set.append(layer.default_style) + + try: + if gs_dataset.styles: + alt_styles = gs_dataset.styles + for alt_style in alt_styles: + if ( + alt_style + and alt_style.name + and alt_style.name != layer.default_style.name + and alt_style.workspace != layer.default_style.workspace + ): + _s, _ = save_style(alt_style, layer) + style_set.append(_s) + except Exception as e: + logger.exception(e) + + if style_set: + # Remove duplicates + style_set = list(dict.fromkeys(style_set)) + layer.styles.set(style_set) + + clean_styles(layer, gs_catalog) + + # Update default style to database + to_update = {"default_style": layer.default_style} + + Dataset.objects.filter(id=layer.id).update(**to_update) + layer.refresh_from_db() + + # Legend links + logger.debug(f" -- Resource Links[Legend link] for layer {layer.name}...") + try: + from geonode.base.models import Link + + dataset_legends = Link.objects.filter(resource=layer.resourcebase_ptr, name="Legend") + for style in set( + list(layer.styles.all()) + + [ + layer.default_style, + ] + ): + if style: + style_name = os.path.basename(urlparse(style.sld_url).path).split(".")[0] + legend_url = get_legend_url(layer, style_name) + if dataset_legends.filter(resource=layer.resourcebase_ptr, name="Legend", url=legend_url).count() < 2: + Link.objects.update_or_create( + resource=layer.resourcebase_ptr, + name="Legend", + url=legend_url, + defaults=dict( + extension="png", + url=legend_url, + mime="image/png", + link_type="image", + ), + ) + logger.debug(" -- Resource Links[Legend link]...done!") + except Exception as e: + logger.debug(f" -- Resource Links[Legend link]...error: {e}") + + try: + from .security import set_geowebcache_invalidate_cache + + set_geowebcache_invalidate_cache(layer.alternate or layer.typename, cat=gs_catalog) + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + + +def save_style(gs_style, layer): + style_name = os.path.basename(urlparse(gs_style.body_href).path).split(".")[0] + sld_name = copy.copy(gs_style.name) + sld_body = copy.copy(gs_style.sld_body) + _gs_style = None + if not gs_style.workspace: + logger.debug(f'save_style: Copying style "{sld_name}" to "{layer.workspace}:{layer.name}') + _gs_style = gs_catalog.create_style(layer.name, sld_body, raw=True, overwrite=True, workspace=layer.workspace) + else: + logger.debug( + f'save_style: Retrieving style "{layer.workspace}:{sld_name}" for layer "{layer.workspace}:{layer.name}' + ) + _gs_style = gs_catalog.get_style(name=sld_name, workspace=layer.workspace) + + style = None + try: + style, _ = Style.objects.get_or_create(name=style_name) + style.workspace = _gs_style.workspace + style.sld_title = _gs_style.sld_title if _gs_style.style_format != "css" and _gs_style.sld_title else sld_name + style.sld_body = _gs_style.sld_body + style.sld_url = _gs_style.body_href + style.save() + except Exception as e: + tb = traceback.format_exc() + logger.debug(tb) + raise e + return (style, _gs_style) + + +def is_dataset_attribute_aggregable(store_type, field_name, field_type): + """ + Decipher whether layer attribute is suitable for statistical derivation + """ + + # must be vector layer + if store_type != "dataStore": + return False + # must be a numeric data type + if field_type not in LAYER_ATTRIBUTE_NUMERIC_DATA_TYPES: + return False + # must not be an identifier type field + if field_name.lower() in {"id", "identifier"}: + return False + + return True + + +def get_attribute_statistics(dataset_name, field): + """ + Generate statistics (range, mean, median, standard deviation, unique values) + for layer attribute + """ + + logger.debug("Deriving aggregate statistics for attribute %s", field) + + if not ogc_server_settings.WPS_ENABLED: + return None + try: + return wps_execute_dataset_attribute_statistics(dataset_name, field) + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + logger.exception("Error generating layer aggregate statistics") + + +def get_wcs_record(instance, retry=True): + wcs = WebCoverageService(f"{ogc_server_settings.LOCATION}wcs", "1.0.0") + key = f"{instance.workspace}:{instance.name}" + logger.debug(wcs.contents) + if key in wcs.contents: + return wcs.contents[key] + else: + msg = f"Dataset '{key}' was not found in WCS service at {ogc_server_settings.public_url}." + if retry: + logger.debug(f"{msg} Waiting a couple of seconds before trying again.") + time.sleep(2) + return get_wcs_record(instance, retry=False) + else: + raise GeoNodeException(msg) + + +def get_coverage_grid_extent(instance): + """ + Returns a list of integers with the size of the coverage + extent in pixels + """ + instance_wcs = get_wcs_record(instance) + grid = instance_wcs.grid + return [(int(h) - int(l) + 1) for h, l in zip(grid.highlimits, grid.lowlimits)] + + +GEOSERVER_LAYER_TYPES = { + "vector": FeatureType.resource_type, + "raster": Coverage.resource_type, +} + + +def cleanup(name, uuid): + """Deletes GeoServer and Catalogue records for a given name. + Useful to clean the mess when something goes terribly wrong. + It also verifies if the Django record existed, in which case + it performs no action. + """ + try: + Dataset.objects.get(name=name) + except Dataset.DoesNotExist: + pass + else: + msg = f"Not doing any cleanup because the layer {name} exists in the Django db." + raise GeoNodeException(msg) + + cat = gs_catalog + gs_store = None + gs_dataset = None + gs_resource = None + # FIXME: Could this lead to someone deleting for example a postgis db + # with the same name of the uploaded file?. + try: + gs_store = cat.get_store(name) + if gs_store is not None: + gs_dataset = cat.get_layer(name) + if gs_dataset is not None: + gs_resource = gs_dataset.resource + else: + gs_dataset = None + gs_resource = None + except FailedRequestError as e: + msg = ("Couldn't connect to GeoServer while cleaning up layer " "[%s] !!", str(e)) + logger.warning(msg) + + if gs_dataset is not None: + try: + cat.delete(gs_dataset) + except Exception: + logger.warning("Couldn't delete GeoServer layer during cleanup()") + if gs_resource is not None: + try: + cat.delete(gs_resource) + except Exception: + msg = "Couldn't delete GeoServer resource during cleanup()" + logger.warning(msg) + if gs_store is not None: + try: + cat.delete(gs_store) + except Exception: + logger.warning("Couldn't delete GeoServer store during cleanup()") + + logger.warning("Deleting dangling Catalogue record for [%s] " "(no Django record to match)", name) + + if "geonode.catalogue" in settings.INSTALLED_APPS: + from geonode.catalogue import get_catalogue + + catalogue = get_catalogue() + catalogue.remove_record(uuid) + logger.warning("Finished cleanup after failed Catalogue/Django " "import for layer: %s", name) + + +def create_geoserver_db_featurestore( + store_type=None, + store_name=None, + author_name="admin", + author_email="admin@geonode.org", + charset="UTF-8", + workspace=None, +): + cat = gs_catalog + dsname = store_name or ogc_server_settings.DATASTORE + # get or create datastore + ds_exists = False + try: + if dsname: + ds = cat.get_store(dsname, workspace=workspace) + else: + return None + if ds is None: + raise FailedRequestError + ds_exists = True + except FailedRequestError: + logger.debug(f"Creating target datastore {dsname}") + ds = cat.create_datastore(dsname, workspace=workspace) + db = ogc_server_settings.datastore_db + db_engine = "postgis" if "postgis" in db["ENGINE"] else db["ENGINE"] + ds.connection_parameters.update( + { + "Evictor run periodicity": 300, + "Estimated extends": "true", + "fetch size": 100000, + "encode functions": "false", + "Expose primary keys": "true", + "validate connections": "true", + "Support on the fly geometry simplification": "false", + "Connection timeout": 10, + "create database": "false", + "Batch insert size": 30, + "preparedStatements": "true", + "min connections": 10, + "max connections": 100, + "Evictor tests per run": 3, + "Max connection idle time": 300, + "Loose bbox": "true", + "Test while idle": "true", + "host": db["HOST"], + "port": db["PORT"] if isinstance(db["PORT"], str) else str(db["PORT"]) or "5432", + "database": db["NAME"], + "user": db["USER"], + "passwd": db["PASSWORD"], + "dbtype": db_engine, + } + ) + + if ds_exists: + ds.save_method = "PUT" + else: + logger.debug("Updating target datastore % s" % dsname) + try: + cat.save(ds) + except FailedRequestError as e: + if "already exists in workspace" not in e.args[0]: + raise e + logger.warning("The store was already present in the workspace selected") + + logger.debug("Reloading target datastore % s" % dsname) + ds = get_store(cat, dsname, workspace=workspace) + assert ds.enabled + + return ds + + +def _create_featurestore(name, data, overwrite=False, charset="UTF-8", workspace=None): + cat = gs_catalog + cat.create_featurestore(name, data, workspace=workspace, overwrite=overwrite, charset=charset) + store = get_store(cat, name, workspace=workspace) + return store, cat.get_resource(name=name, store=store, workspace=workspace) + + +def _create_coveragestore(name, data, overwrite=False, charset="UTF-8", workspace=None): + cat = gs_catalog + cat.create_coveragestore(name, path=data, workspace=workspace, overwrite=overwrite, upload_data=True) + store = get_store(cat, name, workspace=workspace) + return store, cat.get_resource(name=name, store=store, workspace=workspace) + + +def _create_db_featurestore(name, data, overwrite=False, charset="UTF-8", workspace=None): + """Create a database store then use it to import a shapefile. + + If the import into the database fails then delete the store + (and delete the PostGIS table for it). + """ + cat = gs_catalog + db = ogc_server_settings.datastore_db + # dsname = ogc_server_settings.DATASTORE + dsname = db["NAME"] + ds = create_geoserver_db_featurestore(store_name=dsname, workspace=workspace) + + try: + cat.add_data_to_store(ds, name, data, overwrite=overwrite, workspace=workspace, charset=charset) + resource = cat.get_resource(name=name, store=ds, workspace=workspace) + assert resource is not None + return ds, resource + except Exception: + msg = _("An exception occurred loading data to PostGIS") + msg += f"- {sys.exc_info()[1]}" + try: + delete_from_postgis(name, ds) + except Exception: + msg += _(" Additionally an error occured during database cleanup") + msg += f"- {sys.exc_info()[1]}" + raise GeoNodeException(msg) + + +def get_store(cat, name, workspace=None): + # Make sure workspace is a workspace object and not a string. + # If the workspace does not exist, continue as if no workspace had been defined. + if isinstance(workspace, str): + workspace = cat.get_workspace(workspace) + + if workspace is None: + workspace = cat.get_default_workspace() + + if workspace: + try: + store = cat.get_xml(f"{workspace.datastore_url[:-4]}/{name}.xml") + except FailedRequestError: + try: + store = cat.get_xml(f"{workspace.coveragestore_url[:-4]}/{name}.xml") + except FailedRequestError: + try: + store = cat.get_xml(f"{workspace.wmsstore_url[:-4]}/{name}.xml") + except FailedRequestError: + raise FailedRequestError(f"No store found named: {name}") + if store: + if store.tag == "dataStore": + store = datastore_from_index(cat, workspace, store) + elif store.tag == "coverageStore": + store = coveragestore_from_index(cat, workspace, store) + elif store.tag == "wmsStore": + store = wmsstore_from_index(cat, workspace, store) + return store + else: + raise FailedRequestError(f"No store found named: {name}") + else: + raise FailedRequestError(f"No store found named: {name}") + + +def fetch_gs_resource(instance, values, tries): + try: + gs_resource = gs_catalog.get_resource(name=instance.name, store=instance.store, workspace=instance.workspace) + except Exception: + try: + gs_resource = gs_catalog.get_resource( + name=instance.alternate, store=instance.store, workspace=instance.workspace + ) + except Exception: + try: + gs_resource = gs_catalog.get_resource(name=instance.alternate or instance.typename) + except Exception: + gs_resource = None + if gs_resource: + if values: + gs_resource.title = values.get("title", "") + gs_resource.abstract = values.get("abstract", "") + else: + values = {} + + _subtype = gs_resource.store.resource_type + if ( + getattr(gs_resource, "metadata", None) + and gs_resource.metadata.get("time", False) + and gs_resource.metadata.get("time").enabled + ): + _subtype = "vectorTimeSeries" + + values.update( + dict( + store=gs_resource.store.name, + subtype=_subtype, + alternate=f"{gs_resource.store.workspace.name}:{gs_resource.name}", + title=gs_resource.title or gs_resource.store.name, + abstract=gs_resource.abstract or "", + owner=instance.owner, + ) + ) + else: + msg = f"There isn't a geoserver resource for this layer: {instance.name}" + logger.debug(msg) + gs_resource = None + return (values, gs_resource) + + +def wps_execute_dataset_attribute_statistics(dataset_name, field): + """Derive aggregate statistics from WPS endpoint""" + + # generate statistics using WPS + url = urljoin(ogc_server_settings.LOCATION, "ows") + + request = render_to_string("layers/wps_execute_gs_aggregate.xml", {"dataset_name": dataset_name, "field": field}) + u = urlsplit(url) + + headers = { + "User-Agent": "OWSLib (https://geopython.github.io/OWSLib)", + "Content-type": "text/xml", + "Accept": "text/xml", + "Accept-Language": "en-US", + "Accept-Encoding": "gzip,deflate", + "Host": u.netloc, + } + + response, content = http_client.request( + url, method="POST", data=request, headers=headers, user=_user, timeout=5, retries=1 + ) + + exml = dlxml.fromstring(content.encode()) + + result = {} + + for f in ["Min", "Max", "Average", "Median", "StandardDeviation", "Sum"]: + fr = exml.find(f) + if fr is not None: + result[f] = fr.text + else: + result[f] = "NA" + + count = exml.find("Count") + if count is not None: + result["Count"] = int(count.text) + else: + result["Count"] = 0 + + result["unique_values"] = "NA" + + return result + + +def _stylefilterparams_geowebcache_dataset(dataset_name): + headers = {"Content-Type": "text/xml"} + url = f"{ogc_server_settings.LOCATION}gwc/rest/layers/{dataset_name}.xml" + + # read GWC configuration + req, content = http_client.get(url, headers=headers, user=_user) + if req.status_code != 200: + logger.error(f"Error {req.status_code} reading Style Filter Params GeoWebCache at {url}") + return + + # check/write GWC filter parameters + body = None + tree = dlxml.fromstring(_) + param_filters = tree.findall("parameterFilters") + if param_filters and len(param_filters) > 0: + if not param_filters[0].findall("styleParameterFilter"): + style_filters_xml = "STYLES\ + " + style_filters_elem = dlxml.fromstring(style_filters_xml) + param_filters[0].append(style_filters_elem) + body = ET.tostring(tree) + if body: + req, content = http_client.post(url, data=body, headers=headers, user=_user) + if req.status_code != 200: + logger.error(f"Error {req.status_code} writing Style Filter Params GeoWebCache at {url}") + + +def _invalidate_geowebcache_dataset(dataset_name, url=None): + # http.add_credentials(username, password) + headers = { + "Content-Type": "text/xml", + } + body = f""" + {dataset_name} + """.strip() + if not url: + url = f"{ogc_server_settings.LOCATION}gwc/rest/masstruncate" + req, content = http_client.post(url, data=body, headers=headers, user=_user) + + if req.status_code != 200: + logger.debug(f"Error {req.status_code} invalidating GeoWebCache at {url}") + + +def style_update(request, url, workspace=None): + """ + Sync style stuff from GS to GN. + Ideally we should call this from a view straight from GXP, and we should use + gsConfig, that at this time does not support styles updates. Before gsConfig + is updated, for now we need to parse xml. + In case of a DELETE, we need to query request.path to get the style name, + and then remove it. + In case of a POST or PUT, we need to parse the xml from + request.body, which is in this format: + """ + affected_datasets = [] + if request.method in ("POST", "PUT", "DELETE"): # we need to parse xml + # Need to remove NSx from IE11 + if "HTTP_USER_AGENT" in request.META: + if "Trident/7.0" in request.META["HTTP_USER_AGENT"] and "rv:11.0" in request.META["HTTP_USER_AGENT"]: + txml = re.sub(r'xmlns:NS[0-9]=""', "", request.body) + txml = re.sub(r"NS[0-9]:", "", txml) + request._body = txml + style_name = os.path.basename(request.path) + sld_title = style_name + sld_body = None + sld_url = url + dataset_name = None + if "name" in request.GET: + style_name = request.GET["name"] + sld_body = request.body + elif request.method == "DELETE": + style_name = os.path.basename(request.path) + else: + sld_body = request.body + gs_style = gs_catalog.get_style(name=style_name) or gs_catalog.get_style( + name=style_name, workspace=workspace + ) + if gs_style: + sld_title = gs_style.sld_title if gs_style.style_format != "css" and gs_style.sld_title else style_name + sld_body = gs_style.sld_body + sld_url = gs_style.body_href + else: + try: + tree = ET.ElementTree(dlxml.fromstring(request.body)) + elm_nameddataset_name = tree.findall(".//{http://www.opengis.net/sld}Name")[0] + elm_user_style_name = tree.findall(".//{http://www.opengis.net/sld}Name")[1] + elm_user_style_title = tree.find(".//{http://www.opengis.net/sld}Title") + dataset_name = elm_nameddataset_name.text + if elm_user_style_title is None: + sld_title = elm_user_style_name.text + else: + sld_title = elm_user_style_title.text + sld_body = f'{request.body}' + except Exception: + logger.warn("Could not recognize Style and Dataset name from Request!") + + # add style in GN and associate it to layer + if request.method == "DELETE": + if style_name: + Style.objects.filter(name=style_name).delete() + if request.method == "POST": + style = None + if style_name and not re.match(temp_style_name_regex, style_name): + style, created = Style.objects.get_or_create(name=style_name) + style.workspace = workspace + style.sld_body = sld_body + style.sld_url = sld_url + style.sld_title = sld_title + style.save() + layer = None + if dataset_name: + try: + layer = Dataset.objects.get(name=dataset_name) + except Exception: + try: + layer = Dataset.objects.get(alternate=dataset_name) + except Exception: + pass + if layer: + if style: + style.dataset_styles.add(layer) + style.save() + affected_datasets.append(layer) + elif request.method == "PUT": # update style in GN + if style_name and not re.match(temp_style_name_regex, style_name): + style, created = Style.objects.get_or_create(name=style_name) + style.workspace = workspace + style.sld_body = sld_body + style.sld_url = sld_url + style.sld_title = sld_title + style.save() + for layer in style.dataset_styles.all(): + affected_datasets.append(layer) + + # Invalidate GeoWebCache so it doesn't retain old style in tiles + try: + if dataset_name: + _stylefilterparams_geowebcache_dataset(dataset_name) + _invalidate_geowebcache_dataset(dataset_name) + except Exception: + pass + return affected_datasets + + +def set_time_info(layer, attribute, end_attribute, presentation, precision_value, precision_step, enabled=True): + """Configure the time dimension for a layer. + + :param layer: the layer to configure + :param attribute: the attribute used to represent the instant or period + start + :param end_attribute: the optional attribute used to represent the end + period + :param presentation: either 'LIST', 'DISCRETE_INTERVAL', or + 'CONTINUOUS_INTERVAL' + :param precision_value: number representing number of steps + :param precision_step: one of 'seconds', 'minutes', 'hours', 'days', + 'months', 'years' + :param enabled: defaults to True + """ + layer = gs_catalog.get_layer(layer.name) + if layer is None: + raise ValueError(f"no such layer: {layer.name}") + resource = layer.resource if layer else None + if not resource: + resources = gs_catalog.get_resources(stores=[layer.name]) + if resources: + resource = resources[0] + + resolution = None + if precision_value and precision_step: + resolution = f"{precision_value} {precision_step}" + info = DimensionInfo( + "time", enabled, presentation, resolution, "ISO8601", None, attribute=attribute, end_attribute=end_attribute + ) + if resource and hasattr(resource, "metadata") and resource.metadata: + metadata = dict(resource.metadata or {}) + else: + metadata = dict({}) + metadata["time"] = info + + if resource and hasattr(resource, "metadata") and resource.metadata: + resource.metadata = metadata + else: + setattr(resource, "metadata", metadata) + + if resource: + gs_catalog.save(resource) + + +def get_time_info(layer): + """Get the configured time dimension metadata for the layer as a dict. + + The keys of the dict will be those of the parameters of `set_time_info`. + + :returns: dict of values or None if not configured + """ + layer = gs_catalog.get_layer(layer.name) + if layer is None: + raise ValueError(f"no such layer: {layer.name}") + resource = layer.resource if layer else None + if not resource: + resources = gs_catalog.get_resources(stores=[layer.name]) + if resources: + resource = resources[0] + + info = resource.metadata.get("time", None) if resource.metadata else None + vals = None + if info: + value = step = None + resolution = info.resolution_str() + if resolution: + value, step = resolution.split() + vals = dict( + enabled=info.enabled, + attribute=info.attribute, + end_attribute=info.end_attribute, + presentation=info.presentation, + precision_value=value, + precision_step=step, + ) + return vals + + +ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] + +_wms = None +_csw = None +_user, _password = ogc_server_settings.credentials + +url = ogc_server_settings.rest +gs_catalog = Catalog( + url, _user, _password, retries=ogc_server_settings.MAX_RETRIES, backoff_factor=ogc_server_settings.BACKOFF_FACTOR +) +gs_uploader = Client(url, _user, _password) + + +def _create_geofence_client(): + gf_rest_url = f'{url.rstrip("/")}/geofence/' + client = GeoFenceClient(gf_rest_url, _user, _password) + client.set_timeout(ogc_server_settings.GEOFENCE_TIMEOUT) + return client + + +geofence = _create_geofence_client() +gf_utils = GeoFenceUtils(geofence) + +_punc = re.compile(r"[\.:]") # regex for punctuation that confuses restconfig +_foregrounds = ["#ffbbbb", "#bbffbb", "#bbbbff", "#ffffbb", "#bbffff", "#ffbbff"] +_backgrounds = ["#880000", "#008800", "#000088", "#888800", "#008888", "#880088"] +_marks = ["square", "circle", "cross", "x", "triangle"] +_style_contexts = zip(cycle(_foregrounds), cycle(_backgrounds), cycle(_marks)) +_default_style_names = ["point", "line", "polygon", "raster"] +_esri_types = { + "esriFieldTypeDouble": "xsd:double", + "esriFieldTypeString": "xsd:string", + "esriFieldTypeSmallInteger": "xsd:int", + "esriFieldTypeInteger": "xsd:int", + "esriFieldTypeDate": "xsd:dateTime", + "esriFieldTypeOID": "xsd:long", + "esriFieldTypeGeometry": "xsd:geometry", + "esriFieldTypeBlob": "xsd:base64Binary", + "esriFieldTypeRaster": "raster", + "esriFieldTypeGUID": "xsd:string", + "esriFieldTypeGlobalID": "xsd:string", + "esriFieldTypeXML": "xsd:anyType", +} + + +def _dump_image_spec(request_body, image_spec): + millis = int(round(time.time() * 1000)) + try: + with tempfile.TemporaryDirectory() as tmp_dir: + _request_body_file_name = os.path.join(tmp_dir, f"request_body_{millis}.dump") + _image_spec_file_name = os.path.join(tmp_dir, f"image_spec_{millis}.dump") + with open(_request_body_file_name, "w") as _request_body_file: + _request_body_file.write(f"{request_body}") + copyfile(_request_body_file_name, os.path.join(tempfile.gettempdir(), f"request_body_{millis}.dump")) + with open(_image_spec_file_name, "w") as _image_spec_file: + _image_spec_file.write(f"{image_spec}") + copyfile(_image_spec_file_name, os.path.join(tempfile.gettempdir(), f"image_spec_{millis}.dump")) + return f"Dumping image_spec to: {os.path.join(tempfile.gettempdir(), f'image_spec_{millis}.dump')}" + except Exception as e: + logger.exception(e) + return f"Unable to dump image_spec for request: {request_body}" + + +def mosaic_delete_first_granule(cat, layer): + # - since GeoNode will uploade the first granule again through the Importer, we need to / + # delete the one created by the gs_config + cat._cache.clear() + store = cat.get_store(layer) + coverages = cat.mosaic_coverages(store) + + granule_id = f"{layer}.1" + + cat.mosaic_delete_granule(coverages["coverages"]["coverage"][0]["name"], store, granule_id) + + +def set_time_dimension( + cat, + name, + workspace, + time_presentation, + time_presentation_res, + time_presentation_default_value, + time_presentation_reference_value, +): + # configure the layer time dimension as LIST + presentation = time_presentation + if not presentation: + presentation = "LIST" + + resolution = None + if time_presentation == "DISCRETE_INTERVAL": + resolution = time_presentation_res + + strategy = None + if time_presentation_default_value and not time_presentation_default_value == "": + strategy = time_presentation_default_value + + timeInfo = DimensionInfo( + "time", + "true", + presentation, + resolution, + "ISO8601", + None, + attribute="time", + strategy=strategy, + reference_value=time_presentation_reference_value, + ) + + layer = cat.get_layer(name) + resource = layer.resource if layer else None + if not resource: + resources = cat.get_resources(stores=[name]) or cat.get_resources(stores=[name], workspaces=[workspace]) + if resources: + resource = resources[0] + + if not resource: + logger.exception(f"No resource could be found on GeoServer with name {name}") + raise Exception(f"No resource could be found on GeoServer with name {name}") + + resource.metadata = {"time": timeInfo} + cat.save(resource) + + +# main entry point to create a thumbnail - will use implementation +# defined in settings.THUMBNAIL_GENERATOR (see settings.py) +def create_gs_thumbnail(instance, overwrite=False, check_bbox=False): + implementation = import_string(settings.THUMBNAIL_GENERATOR) + return implementation(instance, overwrite, check_bbox) + + +def sync_instance_with_geoserver(instance_id, *args, **kwargs): + """ + Synchronizes the Django Instance with GeoServer layers. + """ + updatebbox = kwargs.get("updatebbox", True) + updatemetadata = kwargs.get("updatemetadata", True) + + instance = None + try: + instance = Dataset.objects.get(id=instance_id) + except Dataset.DoesNotExist: + logger.error(f"Dataset id {instance_id} does not exist yet!") + raise + + if isinstance(instance, ResourceBase): + if hasattr(instance, "dataset"): + instance = instance.dataset + else: + return instance + + try: + instance.set_processing_state("RUNNING") + if updatemetadata: + # Save layer attributes + logger.debug(f"... Refresh GeoServer attributes list for Dataset {instance.title}") + try: + set_attributes_from_geoserver(instance) + except Exception as e: + logger.warning(e) + + # Don't run this signal handler if it is a tile layer or a remote store (Service) + # Currently only gpkg files containing tiles will have this type & will be served via MapProxy. + _is_remote_instance = hasattr(instance, "subtype") and getattr(instance, "subtype") in ["tileStore", "remote"] + + # Let's reset the connections first + gs_catalog._cache.clear() + gs_catalog.reset() + + gs_resource = None + if not _is_remote_instance: + values = {"title": instance.title, "abstract": instance.raw_abstract} + _tries = 0 + _max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 3) + + values, gs_resource = fetch_gs_resource(instance, values, _tries) + while not gs_resource and _tries < _max_tries: + values, gs_resource = fetch_gs_resource(instance, values, _tries) + _tries += 1 + time.sleep(3) + + if gs_resource: + logger.debug(f"Found geoserver resource for this dataset: {instance.name}") + instance.gs_resource = gs_resource + + # Iterate over values from geoserver. + for key in ["alternate", "store", "subtype"]: + # attr_name = key if 'typename' not in key else 'alternate' + # print attr_name + setattr(instance, key, get_dataset_storetype(values[key])) + + if updatemetadata: + # Get metadata links + metadata_links = [] + for link in instance.link_set.metadata(): + metadata_links.append((link.mime, link.name, link.url)) + gs_resource.metadata_links = metadata_links + + """ + TODO: Attributions must be set on a Layer, not a Resource. + We should retrive gs_catalog.get_layer(name=instance.alternate), and obtain the resource from Layer.resource + but I'm not sure if at this stage the layer is ready. For the moment I disable this block. + default_poc = instance.get_first_contact_of_role(role="pointOfContact") + # Update Attribution link + if default_poc: + # gsconfig now utilizes an attribution dictionary + gs_resource.attribution = { + "title": str(instance.poc_csv), + "width": None, + "height": None, + "href": None, + "url": None, + "type": None, + } + profile = get_user_model().objects.get(username=default_poc.username) + site_url = ( + settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + ) + gs_resource.attribution_link = site_url + profile.get_absolute_url() + """ + try: + if settings.RESOURCE_PUBLISHING: + if instance.is_published != gs_resource.advertised: + gs_resource.advertised = "true" + + if any(instance.keyword_list()): + keywords = gs_resource.keywords + instance.keyword_list() + gs_resource.keywords = list(set(keywords)) + + # gs_resource should only be called if + # ogc_server_settings.BACKEND_WRITE_ENABLED == True + if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): + gs_catalog.save(gs_resource) + except Exception as e: + msg = f'Error while trying to save resource named {gs_resource} in GeoServer, try to use: "{e}"' + e.args = (msg,) + logger.warning(e) + + if updatebbox: + # store the resource to avoid another geoserver call in the post_save + """Get information from geoserver. + The attributes retrieved include: + * Bounding Box + * SRID + """ + # This is usually done in Dataset.pre_save, however if the hooks + # are bypassed by custom create/updates we need to ensure the + # bbox is calculated properly. + srid = gs_resource.projection + bbox = gs_resource.native_bbox + ll_bbox = gs_resource.latlon_bbox + try: + instance.set_bbox_polygon([bbox[0], bbox[2], bbox[1], bbox[3]], srid) + except GeoNodeException as e: + if not ll_bbox: + raise + else: + logger.exception(e) + instance.srid = "EPSG:4326" + Dataset.objects.filter(id=instance.id).update(srid=instance.srid) + instance.set_ll_bbox_polygon([ll_bbox[0], ll_bbox[2], ll_bbox[1], ll_bbox[3]]) + + if instance.srid: + instance.srid_url = ( + f"http://www.spatialreference.org/ref/{instance.srid.replace(':', '/').lower()}/" + ) + else: + raise GeoNodeException(_("Invalid Projection. Dataset is missing CRS!")) + + # Update the instance + to_update = {} + if updatemetadata: + to_update = { + "title": instance.title or instance.name, + "abstract": instance.abstract or "", + "alternate": instance.alternate, + } + + if updatebbox and is_monochromatic_image(instance.thumbnail_url): + to_update["thumbnail_url"] = None + + # Save all the modified information in the instance without triggering signals. + with transaction.atomic(): + ResourceBase.objects.filter(id=instance.resourcebase_ptr.id).update(**to_update) + + # to_update['name'] = instance.name, + to_update["workspace"] = gs_resource.store.workspace.name + to_update["store"] = gs_resource.store.name + to_update["subtype"] = instance.subtype + to_update["typename"] = instance.alternate + to_update["srid"] = instance.srid + Dataset.objects.filter(id=instance.id).update(**to_update) + + # Refresh from DB + instance.refresh_from_db() + + if updatemetadata: + # Save dataset styles + logger.debug(f"... Refresh Legend links for Dataset {instance.title}") + try: + set_styles(instance, gs_catalog) + except Exception as e: + logger.warning(e) + + # Invalidate GeoWebCache for the updated resource + try: + _stylefilterparams_geowebcache_dataset(instance.alternate) + _invalidate_geowebcache_dataset(instance.alternate) + except Exception as e: + logger.warning(e) + + # Refreshing dataset links + logger.debug(f"... Creating Default Resource Links for Dataset {instance.title}") + set_resource_default_links(instance, instance, prune=_is_remote_instance) + + # Refreshing CSW records + logger.debug(f"... Updating the Catalogue entries for Dataset {instance.title}") + catalogue_post_save(instance=instance, sender=instance.__class__) + instance.set_processing_state("PROCESSED") + except Exception as e: + logger.exception(e) + instance.set_processing_state("FAILED") + raise GeoNodeException(e) + return instance + + +def get_dataset_storetype(element): + return LAYER_SUBTYPES.get(element, element) + + +def select_relevant_files(allowed_extensions, files): + """Filter the input files list for relevant files only + + Relevant files are those whose extension is in the ``allowed_extensions`` + iterable. + + :param allowed_extensions: list of strings with the extensions to keep + :param files: list of django files with the files to be filtered + """ + result = [] + if files: + for django_file in files: + _django_file_name = django_file if isinstance(django_file, str) else django_file.name + extension = os.path.splitext(_django_file_name)[-1].lower()[1:] + if extension in allowed_extensions: + already_selected = _django_file_name in (f if isinstance(f, str) else f.name for f in result) + if not already_selected: + result.append(django_file) + return result + + +@dataclasses.dataclass() +class SpatialFilesLayerType: + base_file: str + scan_hint: str + spatial_files: typing.List + dataset_type: typing.Optional[str] = None + + +def get_spatial_files_dataset_type(allowed_extensions, files, charset="UTF-8") -> SpatialFilesLayerType: + """Reutnrs 'vector' or 'raster' whether a file from the allowed extensins has been identified.""" + from geonode.upload.files import scan_file + + allowed_file = select_relevant_files(allowed_extensions, files) + if not allowed_file or len(allowed_file) != 1: + return None + base_file = allowed_file[0] + spatial_files = scan_file(base_file, charset=charset) + the_dataset_type = get_dataset_type(spatial_files) + if the_dataset_type not in (FeatureType.resource_type, Coverage.resource_type): + return None + spatial_files_type = SpatialFilesLayerType( + base_file=base_file, + scan_hint=None, + spatial_files=spatial_files, + dataset_type="vector" if the_dataset_type == FeatureType.resource_type else "raster", + ) + + return spatial_files_type + + +def get_dataset_type(spatial_files): + """Returns 'FeatureType.resource_type' or 'Coverage.resource_type' accordingly to the provided SpatialFiles""" + if spatial_files.archive is not None: + the_dataset_type = FeatureType.resource_type + else: + the_dataset_type = spatial_files[0].file_type.dataset_type + return the_dataset_type + + +def get_dataset_capabilities_url(layer, version="1.3.0", access_token=None): + """ + Generate the layer-specific GetCapabilities URL + """ + workspace_layername = layer.alternate.split(":") if ":" in layer.alternate else ("", layer.alternate) + wms_url = settings.GEOSERVER_PUBLIC_LOCATION + if not layer.remote_service: + wms_url = f"{wms_url}{'/'.join(workspace_layername)}/ows?service=wms&version={version}&request=GetCapabilities" # noqa + if access_token: + wms_url += f"&access_token={access_token}" + else: + wms_url = f"{layer.remote_service.service_url}?service=wms&version={version}&request=GetCapabilities" + + return wms_url + + +def get_layer_ows_url(layer, access_token=None): + """ + Generate the layer-specific GetCapabilities URL + """ + workspace_layername = layer.alternate.split(":") if ":" in layer.alternate else ("", layer.alternate) + ows_url = settings.GEOSERVER_PUBLIC_LOCATION + if not layer.remote_service: + ows_url = f"{ows_url}{'/'.join(workspace_layername)}/ows" # noqa + if access_token: + ows_url += f"?access_token={access_token}" + else: + ows_url = f"{layer.remote_service.service_url}" + + return ows_url + + +def wps_format_is_supported(_format, dataset_type): + return (_format, dataset_type) in WPS_ACCEPTABLE_FORMATS + + +def ows_endpoint_in_path(path): + return ( + re.match(r".*(?. +# +######################################################################### diff --git a/geonode/geoserver/management/commands/__init__.py b/geonode/geoserver/management/commands/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/geoserver/management/commands/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/management/commands/find_geoserver_broken_layers.py b/geonode/geoserver/management/commands/find_geoserver_broken_layers.py new file mode 100644 index 0000000..0d9d0cc --- /dev/null +++ b/geonode/geoserver/management/commands/find_geoserver_broken_layers.py @@ -0,0 +1,93 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import sys + +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand + +from geonode.geoserver.helpers import gs_catalog +from geonode.layers.models import Dataset + + +def is_gs_resource_valid(layer): + gs_resource = gs_catalog.get_resource( + name=layer.name, + store=layer.store, + workspace=layer.workspace) + if gs_resource: + return True + else: + return False + + +class Command(BaseCommand): + help = 'Find GeoNode layers with a missing GeoServer resource' + + def add_arguments(self, parser): + parser.add_argument( + '--layername', + dest='layername', + default=None, + help='Filter by a layername.', + ) + parser.add_argument( + '--owner', + dest='owner', + default=None, + help='Filter by a owner.', + ) + parser.add_argument( + '--remove', + action='store_true', + dest='remove', + default=False, + help='Remove a layer if it is broken.', + ) + + def handle(self, **options): + if options['layername']: + layers = Dataset.objects.filter(name__icontains=options['layername']) + else: + layers = Dataset.objects.all() + if options['owner']: + layers = layers.filter(owner=get_user_model().objects.filter(username=options['owner'])) + + layers_count = layers.count() + count = 0 + + dataset_errors = [] + for layer in layers: + count += 1 + try: + print( + f"Checking layer {count}/{layers_count}: {layer.alternate} owned by {layer.owner.username}" + ) + if not is_gs_resource_valid(layer): + print(f"Dataset {layer.alternate} is broken!") + dataset_errors.append(layer) + if options['remove']: + print("Removing this layer...") + layer.delete() + except Exception: + print("Unexpected error:", sys.exc_info()[0]) + + print(f"\n***** Layers with errors: {len(dataset_errors)} in a total of {layers_count} *****") + for dataset_error in dataset_errors: + print(f"{dataset_error.alternate} by {dataset_error.owner.username}") diff --git a/geonode/geoserver/management/commands/importlayers.py b/geonode/geoserver/management/commands/importlayers.py new file mode 100644 index 0000000..4546eda --- /dev/null +++ b/geonode/geoserver/management/commands/importlayers.py @@ -0,0 +1,361 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import time +import json +import datetime +import requests +import traceback + +from urllib.parse import urljoin +from requests.auth import HTTPBasicAuth +from django.conf import settings +from django.utils import timezone +from django.core.management.base import BaseCommand +from geonode.resource.models import ExecutionRequest +from importer.handlers.base import BaseHandler + + +class Command(BaseCommand): + help = ( + "Brings a data file or a directory full of data files into a" + " GeoNode site. Layers are added to the Django database, the" + " GeoServer configuration, and the pycsw metadata index." + ) + + def add_arguments(self, parser): + parser.add_argument("path", nargs="*", help="Path to data file or directory") + parser.add_argument("-hh", "--host", help="Geonode host URL") + parser.add_argument("-u", "--username", help="Geonode username") + parser.add_argument("-p", "--password", help="Geonode password") + parser.add_argument( + "-oe", "--overwrite-existing-layers", type=bool, default=False, + help="Overwrite existing layers" + ) + parser.add_argument( + "-se", "--skip-existing-layers", type=bool, default=False, + help="Skip existing layers (NOT CURRENTLY SUPPORTED IN UPLOAD API)" + ) + parser.add_argument( + "-t", "--tentatives", type=int, default=5, + help="Number of retries for import status check" + ) + + def handle(self, *args, **options): + if not options["path"]: + self.print_help('manage.py', 'importlayers') + return + + host = options.get("host") or getattr( + settings, "SITEURL", "http://localhost:8000" + ) + username = options.get("username") or os.getenv("ADMIN_USERNAME", "admin") + password = options.get("password") or os.getenv("ADMIN_PASSWORD", "admin") + overwrite_existing_layers = options.get("overwrite_existing_layers", False) + skip_existing_layers = options.get("skip_existing_layers", False) + tentatives = options.get("tentatives", 5) + + if skip_existing_layers: + print( + "WARNING: The skip_existing_layers flag is not currently " + "supported in the upload API." + ) + if overwrite_existing_layers: + raise ValueError( + ( + "Both overwrite_existing_layers and skip_existing_layers " + "cannot be true." + ) + ) + + start = datetime.datetime.now(timezone.get_current_timezone()) + + for path in options["path"]: + success, errors = GeoNodeUploader( + host=host, username=username, password=password, + path=path, + overwrite_existing_layers=overwrite_existing_layers, + skip_existing_layers=skip_existing_layers, + tentatives=tentatives + ).execute() + + finish = datetime.datetime.now(timezone.get_current_timezone()) + duration = (finish - start).total_seconds() + + if os.path.isdir(path): + num_files = len(os.listdir(path)) + print(f"{duration / max(num_files, 1)} seconds per layer") + else: + print(f"{duration} seconds for the layer") + + print("Importlayers results:") + print(json.dumps({"success": success, "errors": errors}, indent=4)) + + +class GeoNodeUploader: + + def __init__( + self, host, path, username, password, + overwrite_existing_layers, skip_existing_layers, tentatives=5 + ): + self.host = host + self.path = path + self.username = username + self.password = password + self.overwrite_existing_layers = overwrite_existing_layers + self.skip_existing_layers = skip_existing_layers + self.tentatives = tentatives + self.handlers = BaseHandler.get_registry() + + def execute(self): + success, errors = [], [] + + if os.path.isfile(self.path): + # If path is a single file, process it directly + self.process_file(self.path, success, errors) + elif os.path.isdir(self.path): + # If path is a directory, walk through its files + for root, _, files in os.walk(self.path): + for file in files: + file_path = os.path.join(root, file) + self.process_file(file_path, success, errors) + else: + print(f"Invalid path: {self.path}") + errors.append(f"Invalid path: {self.path}") + + return success, errors + + def process_file(self, file_path, success, errors): + """ + Processes a single file for upload. + + Args: + file_path (str): The full path to the file being processed. + success (list): A list to store successfully processed files. + errors (list): A list to store error messages encountered during processing. + """ + if not os.path.exists(file_path): + print(f"File not found: {file_path}") + errors.append(f"File not found: {file_path}") + return + + handler = self.get_handler(file_path) + if not handler and not self.is_archive_file(file_path): + # Ignore unsupported files + return + + print(f"Processing: {file_path}") + + if handler: + required, optional = self.get_related_files( + handler, file_path, os.path.dirname(file_path) + ) + else: + required, optional = ([], []) + + missing = [f for f in required if not os.path.exists(f)] + if missing: + missing_err_msg = ( + f"{os.path.basename(file_path)}: Missing required files: {missing}" + ) + print(missing_err_msg) + errors.append(missing_err_msg) + return + + params, files_to_upload = self.prepare_upload_params( + os.path.basename(file_path), file_path, required, optional + ) + if self.upload_to_geonode( + params, files_to_upload, os.path.basename(file_path), errors + ): + success.append(os.path.basename(file_path)) + + def get_handler(self, file_path): + """ + Finds the first handler that can process the given file. + + Args: + file_path (str): The full path to the file being checked. + + Returns: + BaseHandler or None: The handler capable of processing the file, + or None if no handler is found. + """ + if self.is_archive_file(file_path): + return None + + _data = {"base_file": file_path} + for handler_class in self.handlers: + if handler_class.can_handle(_data): + return handler_class() + return None + + def is_archive_file(self, file_path): + """ + Checks if the given file is an archive file (e.g., .zip or .kmz) that does not + require a handler. + + Archive files are unpacked by the StorageManager after being sent + to the upload API. + + Args: + file_path (str): The full path to the file being checked. + + Returns: + bool: True if the file is an archive, False otherwise. + """ + file_ext = os.path.splitext(file_path)[1].lower() + return file_ext in [".zip", ".kmz"] + + def get_related_files(self, handler, file_path, root): + """ + Determines the required and optional related files for a given dataset file. + + Args: + handler (BaseHandler): The handler responsible for processing the file. + file_path (str): The full path to the main file. + root (str): The root directory containing the file. + + Returns: + tuple: A list of required file paths and a list of optional file paths. + """ + config = handler.supported_file_extension_config + base, _ = os.path.splitext(os.path.basename(file_path)) + required = [ + os.path.join(root, base + f".{ext}") + for ext in config.get("requires", []) + ] + optional = [ + os.path.join(root, base + f".{ext}") + for ext in config.get("optional", []) + ] + return required, optional + + def prepare_upload_params(self, file, file_path, required, optional): + """ + Prepares the parameters and files required for uploading a dataset to GeoNode. + + Args: + file (str): The name of the main file being uploaded. + file_path (str): The full path to the main file. + required (list): A list of required related file paths. + optional (list): A list of optional related file paths. + + Returns: + tuple: A dictionary of parameters (`params`) and a dictionary of files + to upload (`files_to_upload`). + """ + params = { + "dataset_title": file, + "non_interactive": "true", + "overwrite_existing_layer": str(self.overwrite_existing_layers).lower(), + "skip_existing_layers": str(self.skip_existing_layers).lower(), + } + files_to_upload = {} + + params["base_file"] = os.path.basename(file_path) + files_to_upload["base_file"] = open(file_path, "rb") + + if self.is_archive_file(file_path): + file_ext = os.path.splitext(file_path)[1].lower() + key = file_ext[1:] + "_file" + params[key] = os.path.basename(file_path) + files_to_upload[key] = open(file_path, "rb") + + for f in required + optional: + if os.path.exists(f): + key = os.path.splitext(f)[1][1:] + "_file" + params[key] = os.path.basename(f) + files_to_upload[key] = open(f, "rb") + + return params, files_to_upload + + def upload_to_geonode(self, params, files_to_upload, file, errors): + """ + Uploads a dataset to GeoNode using the upload API. + + Args: + params (dict): A dictionary of parameters required for the upload. + files_to_upload (dict): A dictionary of file objects to be uploaded. + file (str): The name of the main file being uploaded. + errors (list): A list to store error messages encountered during the upload. + + Returns: + bool: True if the upload is successful, False otherwise. + """ + client = requests.session() + print(f"{file}: Submitting to upload API ...") + response = client.post( + urljoin(self.host, "/api/v2/uploads/upload/"), + auth=HTTPBasicAuth(self.username, self.password), + data=params, + files=files_to_upload, + ) + + print(f"{file}: Received status code: {response.status_code}") + + for value in files_to_upload.values(): + value.close() + + if response.status_code in [500, 400, 401, 403]: + upload_error_msg = ( + f"{file}: Error uploading: {response.status_code} - {response.content}" + ) + print(upload_error_msg) + errors.append(upload_error_msg) + return False + + try: + data = response.json() + exec_id = data.get("execution_id") + if not exec_id: + missing_id_msg = f"{file}: Missing execution_id in response" + print(missing_id_msg) + errors.append(missing_id_msg) + return False + + _exec = ExecutionRequest.objects.get(exec_id=exec_id) + retries = 0 + + while ( + _exec.status.lower() in ["ready", "running"] + and retries < self.tentatives + ): + print(f"{file}: Waiting for upload process to finish ...") + time.sleep(10) + _exec.refresh_from_db() + retries += 1 + + if _exec.status.lower() == "running" and retries >= self.tentatives: + timeout_msg = f"{file}: Timeout waiting for upload process to finish" + print(timeout_msg) + errors.append(timeout_msg) + return False + + if _exec.status.lower() != "finished": + error_msg = f"{file}: {_exec.log}" + print(error_msg) + errors.append(error_msg) + return False + + print(f"{file}: Upload completed successfully") + return True + except json.JSONDecodeError: + traceback.print_exc() + return False diff --git a/geonode/geoserver/management/commands/set_all_datasets_public.py b/geonode/geoserver/management/commands/set_all_datasets_public.py new file mode 100644 index 0000000..d4e284a --- /dev/null +++ b/geonode/geoserver/management/commands/set_all_datasets_public.py @@ -0,0 +1,55 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.conf import settings +from django.core.management.base import BaseCommand +from geonode.layers.models import Dataset + +from geonode.geoserver.security.utils import set_geofence_all + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """Resets Permissions to Public for All Layers + """ + + def handle(self, *args, **options): + all_datasets = Dataset.objects.all() + + for index, layer in enumerate(all_datasets): + print(f"[{(index + 1)} / {len(all_datasets)}] Setting public permissions to Dataset [{layer.name}] ...") + try: + use_geofence = settings.OGC_SERVER['default'].get( + "GEOFENCE_SECURITY_ENABLED", False) + if use_geofence: + set_geofence_all(layer) + layer.set_default_permissions() + perm_spec = {"users": {}, "groups": {}} + perm_spec["users"]["admin"] = ['view_resourcebase', 'change_resourcebase_permissions', + 'download_resourcebase', 'publish_resourcebase', + 'change_resourcebase_metadata'] + perm_spec["users"][str(layer.owner)] = ['view_resourcebase', 'change_resourcebase_permissions', + 'download_resourcebase', 'publish_resourcebase', + 'change_resourcebase_metadata'] + perm_spec["users"]["AnonymousUser"] = ['view_resourcebase', 'download_resourcebase'] + layer.set_permissions(perm_spec) + except Exception: + logger.error(f"[ERROR] Dataset [{layer.name}] couldn't be updated") diff --git a/geonode/geoserver/management/commands/set_default_gridsets.py b/geonode/geoserver/management/commands/set_default_gridsets.py new file mode 100644 index 0000000..0826b74 --- /dev/null +++ b/geonode/geoserver/management/commands/set_default_gridsets.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.layers.models import Dataset +from django.core.management.base import BaseCommand +from requests.auth import HTTPBasicAuth +from defusedxml import lxml as dlxml +import xml.etree.ElementTree as ET +from django.conf import settings +from lxml import etree +import requests + + +class Command(BaseCommand): + help = 'Assign all the default values to GeoServer Gridsets' + + def add_arguments(self, parser): + pass + + def handle(self, **options): + try: + url = settings.OGC_SERVER['default']['LOCATION'] + user = settings.OGC_SERVER['default']['USER'] + passwd = settings.OGC_SERVER['default']['PASSWORD'] + """ + curl -v -u admin:geoserver -XGET \ + "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" + """ + layers = Dataset.objects.all() + print(f"Total layers to be updated: {layers.count()}") + for layer in layers: + print(f"Processing layer: {layer.typename}") + r = requests.get(f'{url}gwc/rest/layers/{layer.typename}.xml', + auth=HTTPBasicAuth(user, passwd)) + + if (r.status_code < 200 or r.status_code > 201): + print(f"Dataset does not exists on geoserver: {layer.name}") + continue + try: + xml_content = r.content + tree = dlxml.fromstring(xml_content) + + gwc_id = tree.find('id') + tree.remove(gwc_id) + + gwc_gridSubsets = tree.find('gridSubsets') + if len(gwc_gridSubsets) == 6: + continue + + tree.remove(gwc_gridSubsets) + gwc_gridSubsets = etree.Element('gridSubsets') + for gridSubset in ('EPSG:3857', 'EPSG:3857x2', 'EPSG:4326', 'EPSG:4326x2', 'EPSG:900913', 'EPSG:900913x2'): + gwc_gridSubset = etree.Element('gridSubset') + gwc_gridSetName = etree.Element('gridSetName') + gwc_gridSetName.text = gridSubset + gwc_gridSubset.append(gwc_gridSetName) + gwc_gridSubsets.append(gwc_gridSubset) + + tree.append(gwc_gridSubsets) + + """ + curl -v -u admin:geoserver -XPOST \ + -H "Content-type: text/xml" -d @poi.xml \ + "http://localhost:8080/geoserver/gwc/rest/layers/tiger:poi.xml" + """ + headers = {'Content-type': 'text/xml'} + payload = ET.tostring(tree) + r = requests.post(f'{url}gwc/rest/layers/{layer.typename}.xml', + headers=headers, + data=payload, + auth=HTTPBasicAuth(user, passwd)) + if (r.status_code < 200 or r.status_code > 201): + print(f"Error during update of layer in geoserver: {layer.name} {r.content}") + continue + except Exception as e: + raise e + except Exception as e: + raise e diff --git a/geonode/geoserver/management/commands/sync_geonode_datasets.py b/geonode/geoserver/management/commands/sync_geonode_datasets.py new file mode 100644 index 0000000..abecaf1 --- /dev/null +++ b/geonode/geoserver/management/commands/sync_geonode_datasets.py @@ -0,0 +1,199 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import sys +import json +import logging + +from django.core.management.base import BaseCommand + +from geonode.base.management.command_utils import setup_logger +from geonode.layers.models import Dataset +from geonode.security.views import _perms_info_json +from geonode.base.utils import remove_duplicate_links +from geonode.geoserver.helpers import create_gs_thumbnail, sync_instance_with_geoserver, set_attributes_from_geoserver + +logger = logging.getLogger(__name__) + + +def sync_geonode_datasets( + ignore_errors, + filter, + username, + removeduplicates, + updatepermissions, + updatethumbnails, + updateattributes, + updatebbox, + updatemetadata, +): + layers = Dataset.objects.all().order_by("name") + if filter: + layers = layers.filter(name__icontains=filter) + if username: + layers = layers.filter(owner__username=username) + layers_count = layers.count() + count = 0 + dataset_errors = [] + + if not layers: + logger.warning(f"No layers selected by filter '{filter}'") + return + + for layer in layers: + try: + count += 1 + logger.info(f"=== Syncing layer {count}/{layers_count}: {layer.name}") + if updatepermissions: + logger.info("Syncing permissions...") + # sync permissions in GeoFence + perm_spec = json.loads(_perms_info_json(layer)) + # re-sync GeoFence security rules + layer.set_permissions(perm_spec) + if updateattributes: + # recalculate the layer statistics + logger.info("Setting attributes...") + set_attributes_from_geoserver(layer, overwrite=True) + if updatethumbnails: + logger.info("Regenerating thumbnails...") + create_gs_thumbnail(layer, overwrite=True, check_bbox=False) + if updatebbox: + logger.info("Regenerating BBOX...") + sync_instance_with_geoserver(layer.id, updatemetadata=False, updatebbox=True) + if updatemetadata: + logger.info("Updating metadata...") + sync_instance_with_geoserver(layer.id, updatemetadata=True, updatebbox=False) + if removeduplicates: + # remove duplicates + logger.info("Removing duplicate links...") + remove_duplicate_links(layer) + except (Exception, RuntimeError): + dataset_errors.append(layer.alternate) + exception_type, error, traceback = sys.exc_info() + logger.info(exception_type, error, traceback) + if ignore_errors: + pass + else: + logger.error( + "Stopping process because --ignore-errors was not set and an error was found.", stack_info=True + ) + return + logger.info(f"There are {len(dataset_errors)} layers which could not be updated because of errors") + for dataset_error in dataset_errors: + logger.info(dataset_error) + + +class Command(BaseCommand): + help = "Update the GeoNode layers: permissions (including GeoFence database), statistics, thumbnails" + + def add_arguments(self, parser): + parser.add_argument( + "-i", + "--ignore-errors", + action="store_true", + dest="ignore_errors", + default=False, + help="Stop after any errors are encountered.", + ) + parser.add_argument( + "-d", + "--remove-duplicates", + action="store_true", + dest="removeduplicates", + default=False, + help="Remove duplicates first.", + ) + parser.add_argument( + "-f", + "--filter", + dest="filter", + default=None, + help="Only update data the layers that match the given filter.", + ), + parser.add_argument( + "-u", "--username", dest="username", default=None, help="Only update data owned by the specified username." + ) + parser.add_argument( + "--updatepermissions", + action="store_true", + dest="updatepermissions", + default=False, + help="Update the layer permissions.", + ) + parser.add_argument( + "--updatethumbnails", + action="store_true", + dest="updatethumbnails", + default=False, + help="Update the layer styles and thumbnails.", + ) + parser.add_argument( + "--updateattributes", + action="store_true", + dest="updateattributes", + default=False, + help="Update the layer attributes.", + ) + parser.add_argument( + "--updatebbox", action="store_true", dest="updatebbox", default=False, help="Update the layer BBOX." + ) + parser.add_argument( + "--updatemetadata", + action="store_true", + dest="updatemetadata", + default=False, + help="Update the Geoserver layer metadata.", + ) + parser.add_argument( + "--debug", + action="store_true", + dest="log_debug", + default=False, + help="Enable debug logging.", + ) + + def handle(self, **options): + log_level = logging.DEBUG if options.get("log_debug") else logging.INFO + setup_logger(__name__, level=log_level) + import geonode.geoserver.helpers as helpers + + setup_logger(helpers.__name__, level=log_level) + + ignore_errors = options.get("ignore_errors") + removeduplicates = options.get("removeduplicates") + updatepermissions = options.get("updatepermissions") + updatethumbnails = options.get("updatethumbnails") + updateattributes = options.get("updateattributes") + updatebbox = options.get("updatebbox") + updatemetadata = options.get("updatemetadata") + filter = options.get("filter") + if not options.get("username"): + username = None + else: + username = options.get("username") + sync_geonode_datasets( + ignore_errors, + filter, + username, + removeduplicates, + updatepermissions, + updatethumbnails, + updateattributes, + updatebbox, + updatemetadata, + ) diff --git a/geonode/geoserver/management/commands/sync_geonode_maps.py b/geonode/geoserver/management/commands/sync_geonode_maps.py new file mode 100644 index 0000000..c85b715 --- /dev/null +++ b/geonode/geoserver/management/commands/sync_geonode_maps.py @@ -0,0 +1,135 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import sys + +from django.core.management.base import BaseCommand + +from geonode.maps.models import Map +from geonode.base.utils import remove_duplicate_links +from geonode.geoserver.helpers import ( + create_gs_thumbnail +) + + +def sync_geonode_maps(ignore_errors, + filter, + username, + removeduplicates, + updatethumbnails, + updatebbox, + ): + maps = Map.objects.all().order_by('title') + if filter: + maps = maps.filter(title__icontains=filter) + if username: + maps = maps.filter(owner__username=username) + maps_count = maps.count() + count = 0 + map_errors = [] + for map in maps: + try: + count += 1 + print(f"Syncing map {count}/{maps_count}: {map.title}") + if updatethumbnails: + print("Regenerating thumbnails...") + create_gs_thumbnail(map, overwrite=True, check_bbox=False) + if removeduplicates: + # remove duplicates + print("Removing duplicate links...") + remove_duplicate_links(map) + if updatebbox: + print("Regenerating BBOX...") + map.compute_bbox() + except (Exception, RuntimeError): + map_errors.append(map.title) + exception_type, error, traceback = sys.exc_info() + print(exception_type, error, traceback) + if ignore_errors: + pass + else: + import traceback + traceback.print_exc() + print("Stopping process because --ignore-errors was not set and an error was found.") + return + print(f"There are {len(map_errors)} maps which could not be updated because of errors") + for map_error in map_errors: + print(map_error) + + +class Command(BaseCommand): + help = 'Update the GeoNode maps: permissions, thumbnails' + + def add_arguments(self, parser): + parser.add_argument( + '-i', + '--ignore-errors', + action='store_true', + dest='ignore_errors', + default=False, + help='Stop after any errors are encountered.' + ) + parser.add_argument( + '-d', + '--remove-duplicates', + action='store_true', + dest='removeduplicates', + default=False, + help='Remove duplicates first.' + ) + parser.add_argument( + '-f', + '--filter', + dest="filter", + default=None, + help="Only update data the maps that match the given filter."), + parser.add_argument( + '-u', + '--username', + dest="username", + default=None, + help="Only update data owned by the specified username.") + parser.add_argument( + '--updatethumbnails', + action='store_true', + dest="updatethumbnails", + default=False, + help="Update the map styles and thumbnails.") + parser.add_argument( + "--updatebbox", action="store_true", dest="updatebbox", default=False, help="Update the map BBOX." + ) + + def handle(self, **options): + ignore_errors = options.get('ignore_errors') + removeduplicates = options.get('removeduplicates') + updatethumbnails = options.get('updatethumbnails') + updatebbox = options.get("updatebbox") + filter = options.get('filter') + if not options.get('username'): + username = None + else: + username = options.get('username') + sync_geonode_maps( + ignore_errors, + filter, + username, + removeduplicates, + updatethumbnails, + updatebbox, + ) diff --git a/geonode/geoserver/management/commands/sync_security_rules.py b/geonode/geoserver/management/commands/sync_security_rules.py new file mode 100644 index 0000000..6fb785b --- /dev/null +++ b/geonode/geoserver/management/commands/sync_security_rules.py @@ -0,0 +1,30 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand +from geonode.geoserver.security import sync_resources_with_guardian + + +class Command(BaseCommand): + """ + Sync resources with Guardian and clear their dirty state + """ + + def handle(self, *args, **options): + sync_resources_with_guardian(force=True) diff --git a/geonode/geoserver/management/commands/updatelayers.py b/geonode/geoserver/management/commands/updatelayers.py new file mode 100644 index 0000000..a00d03b --- /dev/null +++ b/geonode/geoserver/management/commands/updatelayers.py @@ -0,0 +1,156 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import ast +import sys +import traceback +from django.core.management.base import BaseCommand +from geonode.people.utils import get_valid_user +from geonode.geoserver.helpers import gs_slurp + + +class Command(BaseCommand): + help = 'Update the GeoNode application with data from GeoServer' + + def add_arguments(self, parser): + parser.add_argument( + '-i', + '--ignore-errors', + action='store_true', + dest='ignore_errors', + default=False, + help='Stop after any errors are encountered.' + ) + parser.add_argument( + '--skip-unadvertised', + action='store_true', + dest='skip_unadvertised', + default=False, + help='Skip processing unadvertised layers from GeoSever.'), + parser.add_argument( + '--skip-geonode-registered', + action='store_true', + dest='skip_geonode_registered', + default=False, + help='Just processing GeoServer layers still not registered in GeoNode.'), + parser.add_argument( + '--remove-deleted', + action='store_true', + dest='remove_deleted', + default=False, + help='Remove GeoNode layers that have been deleted from GeoSever.'), + parser.add_argument( + '-u', + '--user', + dest="user", + default=None, + help="Name of the user account which should own the imported layers"), + parser.add_argument( + '-f', + '--filter', + dest="filter", + default=None, + help="Only update data the layers that match the given filter"), + parser.add_argument( + '-s', + '--store', + dest="store", + default=None, + help="Only update data the layers for the given geoserver store name"), + parser.add_argument( + '-w', + '--workspace', + dest="workspace", + default=None, + help="Only update data on specified workspace"), + parser.add_argument( + '-p', + '--permissions', + dest="permissions", + default=None, + help="Permissions to apply to each layer") + + def handle(self, **options): + ignore_errors = options.get('ignore_errors') + skip_unadvertised = options.get('skip_unadvertised') + skip_geonode_registered = options.get('skip_geonode_registered') + remove_deleted = options.get('remove_deleted') + verbosity = int(options.get('verbosity')) + user = options.get('user') + owner = get_valid_user(user) + workspace = options.get('workspace') + filter = options.get('filter') + store = options.get('store') + if not options.get('permissions'): + permissions = None + else: + permissions = ast.literal_eval(options.get('permissions')) + + if verbosity > 0: + console = sys.stdout + else: + console = None + + output = gs_slurp( + ignore_errors, + verbosity=verbosity, + owner=owner, + console=console, + workspace=workspace, + store=store, + filter=filter, + skip_unadvertised=skip_unadvertised, + skip_geonode_registered=skip_geonode_registered, + remove_deleted=remove_deleted, + permissions=permissions, + execute_signals=True) + + if verbosity > 1: + print("\nDetailed report of failures:") + for dict_ in output['layers']: + if dict_['status'] == 'failed': + print("\n\n", dict_['name'], "\n================") + traceback.print_exception(dict_['exception_type'], + dict_['error'], + dict_['traceback']) + if remove_deleted: + print("Detailed report of layers to be deleted from GeoNode that failed:") + for dict_ in output['deleted_datasets']: + if dict_['status'] == 'delete_failed': + print("\n\n", dict_['name'], "\n================") + traceback.print_exception(dict_['exception_type'], + dict_['error'], + dict_['traceback']) + + if verbosity > 0: + print( + f"\n\nFinished processing {len(output['layers'])} " + f"layers in {output['stats']['duration_sec']:.2f} seconds.\n" + ) + print(f"{output['stats']['created']} Created layers") + print(f"{output['stats']['updated']} Updated layers") + print(f"{output['stats']['failed']} Failed layers") + try: + duration_dataset = round( + output['stats']['duration_sec'] * 1.0 / len(output['layers']), 2) + except ZeroDivisionError: + duration_dataset = 0 + if len(output) > 0: + print(f"{duration_dataset} seconds per layer") + if remove_deleted: + print(f"\n{output['stats']['deleted']} Deleted layers") diff --git a/geonode/geoserver/manager.py b/geonode/geoserver/manager.py new file mode 100644 index 0000000..6b66132 --- /dev/null +++ b/geonode/geoserver/manager.py @@ -0,0 +1,586 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import typing +import logging +import tempfile +import dataclasses + +from gsimporter.api import Session + +from django.conf import settings +from django.db.models.query import QuerySet +from django.contrib.auth.models import Group +from django.contrib.auth import get_user_model + +from geonode.maps.models import Map +from geonode.base import enumerations +from geonode.layers.models import Dataset +from geonode.upload.models import Upload +from geonode.base.models import ResourceBase +from geonode.utils import get_dataset_workspace +from geonode.services.enumerations import CASCADED +from geonode.security.utils import skip_registered_members_common_group +from geonode.security.permissions import ( + VIEW_PERMISSIONS, + OWNER_PERMISSIONS, + DOWNLOAD_PERMISSIONS, + DATASET_ADMIN_PERMISSIONS, +) +from geonode.resource.manager import ResourceManager, ResourceManagerInterface +from geonode.geoserver.signals import geofence_rule_assign +from .geofence import AutoPriorityBatch +from .tasks import geoserver_set_style, geoserver_delete_map, geoserver_create_style, geoserver_cascading_delete +from .helpers import ( + SpatialFilesLayerType, + gs_catalog, + gs_uploader, + set_styles, + set_time_info, + ogc_server_settings, + get_spatial_files_dataset_type, + sync_instance_with_geoserver, + set_attributes_from_geoserver, + create_gs_thumbnail, + create_geoserver_db_featurestore, + geofence, + gf_utils, +) +from .security import ( + _get_gwc_filters_and_formats, + toggle_dataset_cache, + invalidate_geofence_cache, + has_geolimits, + create_geofence_rules, +) + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass() +class GeoServerImporterSessionInfo: + upload_session: Upload + import_session: Session + spatial_files_type: SpatialFilesLayerType + dataset_name: typing.AnyStr + workspace: typing.AnyStr + target_store: typing.AnyStr + + +class GeoServerResourceManager(ResourceManagerInterface): + def search(self, filter: dict, /, resource_type: typing.Optional[object]) -> QuerySet: + return resource_type.objects.none() + + def exists(self, uuid: str, /, instance: ResourceBase = None) -> bool: + if instance: + _real_instance = instance.get_real_instance() + if hasattr(_real_instance, "subtype") and _real_instance.subtype not in ["tileStore", "remote"]: + try: + logger.debug(f"Searching GeoServer for layer '{_real_instance.alternate}'") + # Let's reset the connections first + gs_catalog._cache.clear() + gs_catalog.reset() + if gs_catalog.get_layer(_real_instance.alternate): + return True + except Exception as e: + logger.debug(e) + return False + return True + return False + + def delete(self, uuid: str, /, instance: ResourceBase = None) -> int: + """Removes the layer from GeoServer""" + # cascading_delete should only be called if + # ogc_server_settings.BACKEND_WRITE_ENABLED == True + if instance and getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): + try: + _real_instance = instance.get_real_instance() + if ( + isinstance(_real_instance, Dataset) + and hasattr(_real_instance, "alternate") + and _real_instance.alternate + ): + if ( + not hasattr(_real_instance, "remote_service") + or _real_instance.remote_service is None + or _real_instance.remote_service.method == CASCADED + ): + geoserver_cascading_delete.apply_async(args=(_real_instance.alternate,), expiration=30) + elif isinstance(_real_instance, Map): + geoserver_delete_map.apply_async(args=(_real_instance.id,), expiration=30) + except Exception as e: + logger.exception(e) + + def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, defaults: dict = {}) -> ResourceBase: + _resource = resource_type.objects.get(uuid=uuid) + if resource_type == Dataset: + _synced_resource = sync_instance_with_geoserver(_resource.id) + _resource = _synced_resource or _resource + return _resource + + def update( + self, + uuid: str, + /, + instance: ResourceBase = None, + xml_file: str = None, + metadata_uploaded: bool = False, + vals: dict = {}, + regions: dict = {}, + keywords: dict = {}, + custom: dict = {}, + notify: bool = True, + **kwargs, + ) -> ResourceBase: + if instance: + if isinstance(instance.get_real_instance(), Dataset): + _synced_resource = sync_instance_with_geoserver(instance.id) + instance = _synced_resource or instance + return instance + + def copy( + self, instance: ResourceBase, /, uuid: str = None, owner: settings.AUTH_USER_MODEL = None, defaults: dict = {} + ) -> ResourceBase: + if uuid and instance: + _resource = ResourceManager._get_instance(uuid) + if _resource and isinstance(_resource.get_real_instance(), Dataset): + importer_session_opts = defaults.get("importer_session_opts", {}) + if not importer_session_opts: + _src_upload_session = Upload.objects.filter(resource=instance.get_real_instance().resourcebase_ptr) + if _src_upload_session.exists(): + _src_upload_session = _src_upload_session.get() + if _src_upload_session and _src_upload_session.get_session: + try: + _src_importer_session = _src_upload_session.get_session.import_session.reload() + importer_session_opts.update({"transforms": _src_importer_session.tasks[0].transforms}) + except Exception as e: + logger.exception(e) + return self.import_dataset( + "import_dataset", + uuid, + instance=_resource, + files=defaults.get("files", None), + user=defaults.get("user", _resource.owner), + defaults=defaults, + action_type="create", + importer_session_opts=importer_session_opts, + ) + return _resource + + def append(self, instance: ResourceBase, vals: dict = {}, *args, **kwargs) -> ResourceBase: + if instance and isinstance(instance.get_real_instance(), Dataset): + return self.import_dataset( + "import_dataset", + instance.uuid, + instance=instance, + files=vals.get("files", None), + user=vals.get("user", instance.owner), + action_type="append", + importer_session_opts=vals.get("importer_session_opts", None), + **kwargs, + ) + return instance + + def replace(self, instance: ResourceBase, vals: dict = {}, *args, **kwargs) -> ResourceBase: + if instance and isinstance(instance.get_real_instance(), Dataset): + return self.import_dataset( + "import_dataset", + instance.uuid, + instance=instance, + files=vals.get("files", None), + user=vals.get("user", instance.owner), + action_type="replace", + importer_session_opts=vals.get("importer_session_opts", None), + **kwargs, + ) + return instance + + def import_dataset(self, method: str, uuid: str, /, instance: ResourceBase = None, **kwargs) -> ResourceBase: + instance = instance or ResourceManager._get_instance(uuid) + + if instance and isinstance(instance.get_real_instance(), Dataset): + try: + _gs_import_session_info = self._execute_resource_import( + instance, + kwargs.get("files", None), + kwargs.get("user", instance.owner), + action_type=kwargs.get("action_type", "create"), + importer_session_opts=kwargs.get("importer_session_opts", None), + ) + import_session = _gs_import_session_info.import_session + if import_session: + if import_session.state == enumerations.STATE_PENDING: + task = None + native_crs = None + target_crs = "EPSG:4326" + for _task in import_session.tasks: + # CRS missing/unknown + if _task.state == "NO_CRS": + task = _task + native_crs = _task.layer.srs + break + if not native_crs: + native_crs = "EPSG:4326" + if task: + task.set_srs(native_crs) + + transform = { + "type": "ReprojectTransform", + "source": native_crs, + "target": target_crs, + } + task.remove_transforms([transform], by_field="type", save=False) + task.add_transforms([transform], save=False) + task.save_transforms() + # Starting import process + import_session.commit() + import_session = import_session.reload() + _gs_import_session_info.import_session = import_session + _gs_import_session_info.dataset_name = import_session.tasks[0].layer.name + _name = ( + _gs_import_session_info.dataset_name + if import_session.state == enumerations.STATE_COMPLETE + else "" + ) + _alternate = ( + f"{_gs_import_session_info.workspace}:{_gs_import_session_info.dataset_name}" + if import_session.state == enumerations.STATE_COMPLETE + else "" + ) + _to_update = { + "name": _name, + "title": instance.title or _gs_import_session_info.dataset_name, + "workspace": _gs_import_session_info.workspace, + "alternate": _alternate, + "typename": _alternate, + "store": _gs_import_session_info.target_store or _gs_import_session_info.dataset_name, + "subtype": _gs_import_session_info.spatial_files_type.dataset_type, + } + if "defaults" in kwargs: + kwargs["defaults"].update(_to_update) + Dataset.objects.filter(uuid=instance.uuid).update(**_to_update) + instance.get_real_instance_class().objects.filter(uuid=instance.uuid).update(**_to_update) + # Refresh from DB + instance.refresh_from_db() + if kwargs.get("action_type", "create") == "create": + set_styles(instance.get_real_instance(), gs_catalog) + set_attributes_from_geoserver(instance.get_real_instance(), overwrite=True) + elif kwargs.get("action_type", "create") == "create": + logger.exception(Exception(f"Importer Session not valid - STATE: {import_session.state}")) + if import_session.state == enumerations.STATE_COMPLETE: + instance.set_processing_state(enumerations.STATE_PROCESSED) + else: + instance.set_processing_state(import_session.state) + instance.set_dirty_state() + instance.save(notify=False) + except Exception as e: + logger.exception(e) + if kwargs.get("action_type", "create") == "create": + instance.delete() + instance = None + return instance + + def _execute_resource_import( + self, instance, files: list, user, action_type: str, importer_session_opts: typing.Optional[typing.Dict] = None + ): + from geonode.utils import get_allowed_extensions + + ALLOWED_EXTENSIONS = get_allowed_extensions() + + session_opts = dict(importer_session_opts) if importer_session_opts is not None else {} + + spatial_files_type = get_spatial_files_dataset_type(ALLOWED_EXTENSIONS, files) + + if not spatial_files_type: + raise Exception(f"No suitable Spatial Files avaialable for 'ALLOWED_EXTENSIONS' = {ALLOWED_EXTENSIONS}.") + + upload_session, _ = Upload.objects.get_or_create( + resource=instance.get_real_instance().resourcebase_ptr, user=user + ) + upload_session.resource = instance.get_real_instance().resourcebase_ptr + upload_session.save() + + _name = instance.get_real_instance().name + if not _name: + _name = ( + session_opts.get("name", None) or os.path.splitext(os.path.basename(spatial_files_type.base_file))[0] + ) + instance.get_real_instance().name = _name + + gs_dataset = None + try: + gs_dataset = gs_catalog.get_layer(_name) + except Exception as e: + logger.debug(e) + + _workspace = None + _target_store = None + if gs_dataset: + _target_store = gs_dataset.resource.store.name if instance.get_real_instance().subtype == "vector" else None + _workspace = gs_dataset.resource.workspace.name if gs_dataset.resource.workspace else None + + if not _workspace: + _workspace = session_opts.get("workspace", instance.get_real_instance().workspace) + if not _workspace: + _workspace = instance.get_real_instance().workspace or settings.DEFAULT_WORKSPACE + + if not _target_store: + if instance.get_real_instance().subtype == "vector" or spatial_files_type.dataset_type == "vector": + _dsname = ogc_server_settings.datastore_db["NAME"] + _ds = create_geoserver_db_featurestore(store_name=_dsname, workspace=_workspace) + if _ds: + _target_store = session_opts.get("target_store", None) or _dsname + + # opening Import session for the selected layer + # Let's reset the connections first + gs_catalog._cache.clear() + gs_catalog.reset() + # Let's now try the new ingestion + import_session = gs_uploader.start_import(import_id=upload_session.id, name=_name, target_store=_target_store) + + upload_session.set_processing_state(enumerations.STATE_PROCESSED) + upload_session.import_id = import_session.id + upload_session.name = _name + upload_session.complete = True + upload_session.processed = True + upload_session.save() + + _gs_import_session_info = GeoServerImporterSessionInfo( + upload_session=upload_session, + import_session=import_session, + spatial_files_type=spatial_files_type, + dataset_name=None, + workspace=_workspace, + target_store=_target_store, + ) + + import_session.upload_task(files) + task = import_session.tasks[0] + # Changing layer name, mode and target + task.layer.set_target_layer_name(_name) + task.set_update_mode(action_type.upper()) + task.set_target(store_name=_target_store, workspace=_workspace) + transforms = session_opts.get("transforms", None) + if transforms: + task.set_transforms(transforms) + # Starting import process + import_session.commit() + import_session = import_session.reload() + + _gs_import_session_info.import_session = import_session + _gs_import_session_info.dataset_name = import_session.tasks[0].layer.name + + return _gs_import_session_info + + def remove_permissions(self, uuid: str, /, instance: ResourceBase = None) -> bool: + instance = instance or ResourceManager._get_instance(uuid) + + try: + if instance and isinstance(instance.get_real_instance(), Dataset): + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: + if not getattr(settings, "DELAYED_SECURITY_SIGNALS", False): + workspace = get_dataset_workspace(instance) + removed = gf_utils.delete_layer_rules(workspace, instance.name) + if removed: + invalidate_geofence_cache() + else: + instance.set_dirty_state() + except Exception as e: + logger.exception(e) + return False + return True + + def set_permissions( + self, + uuid: str, + /, + instance: ResourceBase = None, + owner: settings.AUTH_USER_MODEL = None, + permissions: dict = {}, + created: bool = False, + approval_status_changed: bool = False, + group_status_changed: bool = False, + ) -> bool: + _resource = instance or ResourceManager._get_instance(uuid) + + try: + if _resource: + _resource = _resource.get_real_instance() + logger.info(f'Requesting GeoFence rules on resource "{_resource}" :: {type(_resource).__name__}') + if isinstance(_resource, Dataset): + if settings.OGC_SERVER["default"].get("GEOFENCE_SECURITY_ENABLED", False) or getattr( + settings, "GEOFENCE_SECURITY_ENABLED", False + ): + if not getattr(settings, "DELAYED_SECURITY_SIGNALS", False): + batch = AutoPriorityBatch( + gf_utils.get_first_available_priority(), f"Set permission for resource {_resource}" + ) + + workspace = get_dataset_workspace(_resource) + + if not created: + gf_utils.collect_delete_layer_rules(workspace, _resource.name, batch) + + exist_geolimits = None + _owner = owner or _resource.owner + + if permissions is not None and len(permissions): + # Owner + perms = ( + OWNER_PERMISSIONS.copy() + + DATASET_ADMIN_PERMISSIONS.copy() + + DOWNLOAD_PERMISSIONS.copy() + ) + create_geofence_rules(_resource, perms, _owner, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _owner, None) + + deferred_anon_perms = [] + + # All the other users + if "users" in permissions and len(permissions["users"]) > 0: + for user, user_perms in permissions["users"].items(): + _user = get_user_model().objects.get(username=user) + if _user != _owner: + if user == "AnonymousUser": + _user = None + deferred_anon_perms.append(user_perms) + else: + create_geofence_rules(_resource, user_perms, _user, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _user, None) + + # All the other groups + if "groups" in permissions and len(permissions["groups"]) > 0: + for group, perms in permissions["groups"].items(): + _group = Group.objects.get(name=group) + if _group and _group.name and _group.name == "anonymous": + _group = None + deferred_anon_perms.append(perms) + else: + create_geofence_rules(_resource, perms, None, _group, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, _group) + + for perm in deferred_anon_perms: + create_geofence_rules(_resource, perm, None, None, batch) + + else: + # Owner & Managers + perms = ( + OWNER_PERMISSIONS.copy() + + DATASET_ADMIN_PERMISSIONS.copy() + + DOWNLOAD_PERMISSIONS.copy() + ) + create_geofence_rules(_resource, perms, _owner, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _owner, None) + + _resource_groups, _group_managers = _resource.get_group_managers(group=_resource.group) + for _group_manager in _group_managers: + create_geofence_rules(_resource, perms, _group_manager, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, _group_manager, None) + + for user_group in _resource_groups: + if not skip_registered_members_common_group(user_group): + create_geofence_rules(_resource, perms, None, user_group, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, user_group) + + # Anonymous + if settings.DEFAULT_ANONYMOUS_VIEW_PERMISSION: + create_geofence_rules(_resource, VIEW_PERMISSIONS, None, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, None) + + if settings.DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION: + create_geofence_rules(_resource, DOWNLOAD_PERMISSIONS, None, None, batch) + exist_geolimits = exist_geolimits or has_geolimits(_resource, None, None) + + if exist_geolimits is not None: + filters, formats = _get_gwc_filters_and_formats(exist_geolimits) + try: + _dataset_workspace = get_dataset_workspace(_resource) + toggle_dataset_cache( + f"{_dataset_workspace}:{_resource.name}", filters=filters, formats=formats + ) + except Dataset.DoesNotExist: + pass + + try: + logger.info( + f"Pushing {batch.length()} " f"changes into GeoFence for resource {_resource.name}" + ) + executed = geofence.run_batch(batch) + if executed: + geofence.invalidate_cache() + except Exception as e: + logger.warning( + f"Could not sync GeoFence for resource {_resource}: {e}." " Retrying async." + ) + _resource.set_dirty_state() + else: + _resource.set_dirty_state() + except Exception as e: + logger.exception(e) + return False + + geofence_rule_assign.send_robust(sender=instance, instance=instance) + + return True + + def set_thumbnail( + self, uuid: str, /, instance: ResourceBase = None, overwrite: bool = True, check_bbox: bool = True + ) -> bool: + if instance and ( + isinstance(instance.get_real_instance(), Dataset) or isinstance(instance.get_real_instance(), Map) + ): + if overwrite or not instance.thumbnail_url: + create_gs_thumbnail(instance.get_real_instance(), overwrite=overwrite, check_bbox=check_bbox) + return True + return False + + def exec(self, method: str, uuid: str, /, instance: ResourceBase = None, **kwargs) -> ResourceBase: + raise NotImplementedError + + def set_style(self, method: str, uuid: str, instance: ResourceBase = None, **kwargs) -> ResourceBase: + instance = instance or ResourceManager._get_instance(uuid) + + if instance and isinstance(instance.get_real_instance(), Dataset): + try: + logger.info(f"Creating style for Dataset {instance.get_real_instance()} / {kwargs}") + _sld_file = kwargs.get("sld_file", None) + _tempdir = kwargs.get("tempdir", tempfile.gettempdir()) + if _sld_file and kwargs.get("sld_uploaded", False): + geoserver_set_style(instance.get_real_instance().id, _sld_file) + else: + geoserver_create_style( + instance.get_real_instance().id, instance.get_real_instance().name, _sld_file, _tempdir + ) + except Exception as e: + logger.exception(e) + return None + return instance + + def set_time_info(self, method: str, uuid: str, /, instance: ResourceBase = None, **kwargs) -> ResourceBase: + instance = instance or ResourceManager._get_instance(uuid) + + if instance and isinstance(instance.get_real_instance(), Dataset): + try: + if kwargs.get("time_info", None): + set_time_info(instance.get_real_instance(), **kwargs["time_info"]) + except Exception as e: + logger.exception(e) + return None + return instance diff --git a/geonode/geoserver/migrations/__init__.py b/geonode/geoserver/migrations/__init__.py new file mode 100644 index 0000000..68c7b7d --- /dev/null +++ b/geonode/geoserver/migrations/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/models.py b/geonode/geoserver/models.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/geoserver/models.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/ows.py b/geonode/geoserver/ows.py new file mode 100644 index 0000000..fc0e3b3 --- /dev/null +++ b/geonode/geoserver/ows.py @@ -0,0 +1,369 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import ast +import typing +import logging + +from django.conf import settings +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ + +from lxml import etree +from urllib.parse import urlencode, urljoin +from geonode.utils import XML_PARSER, http_client, OGC_Servers_Handler + +logger = logging.getLogger(__name__) + +ogc_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] + +DEFAULT_EXCLUDE_FORMATS = ["PNG", "JPEG", "GIF", "TIFF"] + + +def _get_nsmap(original: typing.Dict, key: str): + """Prepare namespaces dict for running xpath queries. + + lxml complains when a namespaces dict has an entry with None as a key. + + """ + + result = original.copy() + try: + result[key] = original[None] + except KeyError: + pass + else: + del result[None] + + return result + + +def _wcs_get_capabilities(): + try: + wcs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) + except Exception: + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wcs_url += "&" if "?" in wcs_url else "?" + + return wcs_url + urlencode( + { + "service": "WCS", + "request": "GetCapabilities", + "version": "2.0.1", + } + ) + + +def _wcs_describe_coverage(coverage_id): + try: + wcs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) + except Exception: + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wcs_url += "&" if "?" in wcs_url else "?" + return wcs_url + urlencode( + {"service": "WCS", "request": "describecoverage", "version": "2.0.1", "coverageid": coverage_id} + ) + + +def _get_wcs_axis_labels(coverage_id): + """ + This is a utiliy method using the GeoNode Proxy to fetch the "DescribeCoverage". + The outcome will be used to fetch the "Envelope" "axisLabels", that the WCS declares accordingly to the provided "outputCrs". + + The response will be something like the following here below: + + + + + Generated from WorldImage + gebco_2020_n54.0659179519862_s51.64453104883433_w0.5559082031249998_e4.409911975264549 + + + -0.5 -0.5 + 580.5 924.5 + + + geonode__gebco_2020_n54_0659179519862_s51_64453104883433_w0_5559082031249998_e4_409911975264549 + + + Linear + 0 0 + + + + + + gebco_2020_n54.0659179519862_s51.64453104883433_w0.5559082031249998_e4.409911975264549 + WCS + WorldImage + + + + + + + + + + + + + + + 0 0 + 924 580 + + + i j + + + 0.0 0.0 + + + 0.0 1.0 + 1.0 0.0 + + + + + + + GRAY_INDEX + + + + 0 65535 + + + + + + + + RectifiedGridCoverage + image/tiff + + + + """ + + def _swap(_axis_labels): + # WCS 2.0.1 swaps the order of the Lon/Lat axis to Lat/Lon. + if _axis_labels[0].lower() in "lat": + return [_axis_labels[1], _axis_labels[0]] + else: + return _axis_labels + + try: + _describe_coverage_response, _content = http_client.get(_wcs_describe_coverage(coverage_id)) + _describe_coverage_response.raise_for_status() + _root = etree.fromstring(_content.encode("UTF-8"), parser=XML_PARSER) + _nsmap = _get_nsmap(_root.nsmap, "wcs") + _coverage_descriptions = _root.xpath("//wcs:CoverageDescription", namespaces=_nsmap) + for _coverage_description in _coverage_descriptions: + _envelope = _coverage_description.xpath("gml:boundedBy/gml:Envelope", namespaces=_nsmap) + _axis_labels = _envelope[0].attrib["axisLabels"].split(" ") + if _axis_labels and isinstance(_axis_labels, list) and len(_axis_labels) == 2: + return _swap(_axis_labels) + except Exception as e: + logger.error(e) + return None + + +def _wcs_link(wcs_url, identifier, mime, srid=None, bbox=None, compression=None, tile_size=None): + """ + Utility method generating a "GetCoverage" URL by fixing some parameters, like the "axisLabels" + + e.g.: + http://localhost:8080/geoserver/ows? + service=WCS& + request=GetCoverage& + coverageid=geonode__relief_san_andres0& + format=image%2Ftiff& + version=2.0.1& + compression=DEFLATE& + tileWidth=512& + tileHeight=512& + outputCrs=EPSG%3A4326& + subset=Long(-84.0,-78.0)& + subset=Lat(11.0,15.0) + """ + coverage_id = identifier.replace(":", "__", 1) + wcs_params = { + "service": "WCS", + "request": "GetCoverage", + "coverageid": coverage_id, + "format": mime, + "version": "2.0.1", + } + + if compression: + wcs_params["compression"] = compression + + if tile_size: + wcs_params["tileWidth"] = tile_size + wcs_params["tileHeight"] = tile_size + + if srid: + wcs_params["outputCrs"] = srid + + _wcs_params = urlencode(wcs_params) + + if bbox: + _bbox = None + if isinstance(bbox, list): + _bbox = bbox + elif isinstance(bbox, str): + _bbox = ( + ast.literal_eval(f"[{bbox}]") + if all([_x in bbox for _x in ["[", "]"]]) + else ast.literal_eval(f"[{bbox}]") + ) + if _bbox: + _axis_labels = _get_wcs_axis_labels(coverage_id) + if _axis_labels: + _wcs_params += ( + f"&subset={_axis_labels[0]}({_bbox[0]},{_bbox[2]})&subset={_axis_labels[1]}({_bbox[1]},{_bbox[3]})" + ) + _query_separator = "?" if not wcs_url.endswith("?") else "" + return f"{wcs_url}{_query_separator}{_wcs_params}" + + +def wcs_links(wcs_url, identifier, bbox=None, srid=None): + """ + Creates the WCS GetCoverage Default download links. + + By providing 'None' bbox and srid, we are going to ask to the WCS to + skip subsetting, i.e. output the whole coverage in the netive SRS. + + Notice that the "wcs_links" method also generates 1 default "outputFormat": + - "geotiff"; GeoTIFF which will be compressed and tiled by passing to the WCS the default query params compression='DEFLATE' and tile_size=512 + """ + types = [ + # AF: Slow outputFormat, removed -> ("x-gzip", _("GZIP"), "application/x-gzip", None, None), + ("geotiff", _("GeoTIFF"), "image/tiff", "DEFLATE", 512), + ] + output = [] + for ext, name, mime, compression, tile_size in types: + url = _wcs_link(wcs_url, identifier, mime, bbox=bbox, srid=srid, compression=compression, tile_size=tile_size) + output.append((ext, name, mime, url)) + return output + + +def _wfs_get_capabilities(): + try: + wfs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) + except Exception: + wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wfs_url += "&" if "?" in wfs_url else "?" + + return wfs_url + urlencode( + { + "service": "WFS", + "request": "GetCapabilities", + "version": "1.1.0", + } + ) + + +def _wfs_link(wfs_url, identifier, mime, extra_params, bbox=None, srid=None): + params = { + "service": "WFS", + "version": "1.0.0", + "request": "GetFeature", + "typename": identifier, + "outputFormat": mime, + } + if srid: + params["srs"] = srid + if bbox: + params["bbox"] = bbox + params.update(extra_params) + _query_separator = "?" if not wfs_url.endswith("?") else "" + return f"{wfs_url}{_query_separator}{urlencode(params)}" + + +def wfs_links(wfs_url, identifier, bbox=None, srid=None): + types = [ + ("zip", _("Zipped Shapefile"), "SHAPE-ZIP", {"format_options": "charset:UTF-8"}), + ("gml", _("GML 2.0"), "gml2", {}), + ("gml", _("GML 3.1.1"), "text/xml; subtype=gml/3.1.1", {}), + ("csv", _("CSV"), "csv", {}), + ("excel", _("Excel"), "excel", {}), + ("json", _("GeoJSON"), "json", {"srsName": srid or "EPSG:4326"}), + ] + output = [] + for ext, name, mime, extra_params in types: + url = _wfs_link(wfs_url, identifier, mime, extra_params, bbox=bbox, srid=srid) + output.append((ext, name, mime, url)) + return output + + +def _wms_get_capabilities(): + try: + wms_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) + except Exception: + wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + wms_url += "&" if "?" in wms_url else "?" + + return wms_url + urlencode( + { + "service": "WMS", + "request": "GetCapabilities", + "version": "1.3.0", + } + ) + + +def _wms_link(wms_url, identifier, mime, height, width, srid=None, bbox=None): + wms_params = { + "service": "WMS", + "request": "GetMap", + "layers": identifier, + "format": mime, + "height": height, + "width": width, + } + if srid: + wms_params["srs"] = srid + if bbox: + wms_params["bbox"] = bbox + _query_separator = "?" if not wms_url.endswith("?") else "" + return f"{wms_url}{_query_separator}{urlencode(wms_params)}" + + +def wms_links(wms_url, identifier, bbox, srid, height, width): + types = [ + ("jpg", _("JPEG"), "image/jpeg"), + ("pdf", _("PDF"), "application/pdf"), + ("png", _("PNG"), "image/png"), + ] + output = [] + for ext, name, mime in types: + url = _wms_link(wms_url, identifier, mime, height, width, srid, bbox) + output.append((ext, name, mime, url)) + return output diff --git a/geonode/geoserver/processing/__init__.py b/geonode/geoserver/processing/__init__.py new file mode 100644 index 0000000..6b4db60 --- /dev/null +++ b/geonode/geoserver/processing/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/processing/admin.py b/geonode/geoserver/processing/admin.py new file mode 100644 index 0000000..5e5942a --- /dev/null +++ b/geonode/geoserver/processing/admin.py @@ -0,0 +1,26 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.contrib import admin + +from .models import SampleGeoServerProcessingTask + + +@admin.register(SampleGeoServerProcessingTask) +class SampleGeoServerProcessingTaskAdmin(admin.ModelAdmin): + pass diff --git a/geonode/geoserver/processing/apps.py b/geonode/geoserver/processing/apps.py new file mode 100644 index 0000000..8b5d10c --- /dev/null +++ b/geonode/geoserver/processing/apps.py @@ -0,0 +1,24 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + + +class GeoNodeGeoserverProcessingAppConfig(AppConfig): + name = "geonode.geoserver.processing" + verbose_name = "GeoNode Geoserver Processing" diff --git a/geonode/geoserver/processing/migrations/0001_initial.py b/geonode/geoserver/processing/migrations/0001_initial.py new file mode 100644 index 0000000..b09aa5c --- /dev/null +++ b/geonode/geoserver/processing/migrations/0001_initial.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.4 on 2021-06-23 16:34 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('geonode_resource_processing', '0002_alter_processingworkflowtasks_options'), + ] + + operations = [ + migrations.CreateModel( + name='SampleGeoServerProcessingTask', + fields=[ + ('abstractprocessingtask_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='geonode_resource_processing.abstractprocessingtask')), + ], + options={ + 'abstract': False, + 'base_manager_name': 'objects', + }, + bases=('geonode_resource_processing.abstractprocessingtask',), + ), + ] diff --git a/geonode/geoserver/processing/migrations/__init__.py b/geonode/geoserver/processing/migrations/__init__.py new file mode 100644 index 0000000..6b4db60 --- /dev/null +++ b/geonode/geoserver/processing/migrations/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/processing/models.py b/geonode/geoserver/processing/models.py new file mode 100644 index 0000000..8540349 --- /dev/null +++ b/geonode/geoserver/processing/models.py @@ -0,0 +1,27 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.resource.processing.models import AbstractProcessingTask, AbstractProcessingTaskMeta + + +# ##################################################################################### # +# Samples +# ##################################################################################### # +class SampleGeoServerProcessingTask(AbstractProcessingTask, metaclass=AbstractProcessingTaskMeta): + def execute(self, resource): + print(f"Executing {self.name} against {resource}") diff --git a/geonode/geoserver/processing/tests.py b/geonode/geoserver/processing/tests.py new file mode 100644 index 0000000..6b4db60 --- /dev/null +++ b/geonode/geoserver/processing/tests.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/security.py b/geonode/geoserver/security.py new file mode 100644 index 0000000..4cb244e --- /dev/null +++ b/geonode/geoserver/security.py @@ -0,0 +1,481 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import requests +import traceback +from packaging import version +import re +import typing +import xml.etree.ElementTree as ET + +from lxml import etree +from defusedxml import lxml as dlxml +from requests.auth import HTTPBasicAuth +from guardian.shortcuts import get_anonymous_user + +from django.conf import settings +from django.contrib.auth.models import Group +from django.contrib.auth import get_user_model +from geonode.geoserver.geofence import Batch, Rule, AutoPriorityBatch +from geonode.geoserver.helpers import geofence, gf_utils, gs_catalog +from geonode.groups.models import GroupProfile +from geonode.utils import get_dataset_workspace + + +logger = logging.getLogger(__name__) + + +def delete_all_geofence_rules(): + """purge all existing GeoFence Rules""" + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] or getattr( + settings, "GEOFENCE_SECURITY_ENABLED", False + ): + gf_utils.delete_all_rules() + + +def delete_geofence_rules_for_layer(instance): + """Delete all rules for a given layer + This function wraps the gf_util method, since it doesn't know about Dataset model + """ + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] or getattr( + settings, "GEOFENCE_SECURITY_ENABLED", False + ): + resource = instance.get_self_resource() + workspace_name = get_dataset_workspace(resource.dataset) + layer_name = ( + resource.dataset.name + if resource.dataset and hasattr(resource.dataset, "name") + else resource.dataset.alternate + ) + logger.debug(f"Removing rules for layer {workspace_name}:{layer_name}") + gf_utils.delete_layer_rules(workspace_name, layer_name) + + +def invalidate_geofence_cache(): + """invalidate GeoFence Cache Rules""" + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: + try: + geofence.invalidate_cache() + return True + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + return False + + +def toggle_dataset_cache(dataset_name, enable=True, filters=None, formats=None): + """Disable/enable a GeoServer Tiled Dataset Configuration""" + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: + try: + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] + """ + curl -v -u admin:geoserver -XGET \ + "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" + """ + r = requests.get(f"{url}gwc/rest/layers/{dataset_name}.xml", auth=HTTPBasicAuth(user, passwd)) + + if r.status_code < 200 or r.status_code > 201: + logger.debug(f"Could not Retrieve {dataset_name} Cache.") + return False + try: + xml_content = r.content + tree = dlxml.fromstring(xml_content) + + gwc_id = tree.find("id") + tree.remove(gwc_id) + + gwc_enabled = tree.find("enabled") + if gwc_enabled is None: + gwc_enabled = etree.Element("enabled") + tree.append(gwc_enabled) + gwc_enabled.text = str(enable).lower() + + gwc_mimeFormats = tree.find("mimeFormats") + # Returns an element instance or None + if gwc_mimeFormats is not None and len(gwc_mimeFormats): + tree.remove(gwc_mimeFormats) + + if formats is not None: + for format in formats: + gwc_format = etree.Element("string") + gwc_format.text = format + gwc_mimeFormats.append(gwc_format) + + tree.append(gwc_mimeFormats) + + gwc_parameterFilters = tree.find("parameterFilters") + if filters is None: + tree.remove(gwc_parameterFilters) + else: + for filter in filters: + for k, v in filter.items(): + """ + + + STYLES + + + + """ + gwc_parameter = etree.Element(k) + for parameter_key, parameter_value in v.items(): + gwc_parameter_key = etree.Element("key") + gwc_parameter_key.text = parameter_key + gwc_parameter_value = etree.Element("defaultValue") + gwc_parameter_value.text = parameter_value + + gwc_parameter.append(gwc_parameter_key) + gwc_parameter.append(gwc_parameter_value) + gwc_parameterFilters.append(gwc_parameter) + + """ + curl -v -u admin:geoserver -XPOST \ + -H "Content-type: text/xml" -d @poi.xml \ + "http://localhost:8080/geoserver/gwc/rest/layers/tiger:poi.xml" + """ + headers = {"Content-type": "text/xml"} + payload = ET.tostring(tree) + r = requests.post( + f"{url}gwc/rest/layers/{dataset_name}.xml", + headers=headers, + data=payload, + auth=HTTPBasicAuth(user, passwd), + ) + if r.status_code < 200 or r.status_code > 201: + logger.debug(f"Could not Update {dataset_name} Cache.") + return False + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + return False + return True + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + return False + + +def delete_dataset_cache(dataset_name): + """Delete a GeoServer Tiled Dataset Configuration and all the cache""" + if settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"]: + try: + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] + """ + curl -v -u admin:geoserver -XDELETE \ + "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" + """ + r = requests.delete(f"{url}gwc/rest/layers/{dataset_name}.xml", auth=HTTPBasicAuth(user, passwd)) + + if r.status_code < 200 or r.status_code > 201: + logger.debug(f"Could not Delete {dataset_name} Cache.") + return False + return True + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + return False + + +def set_geowebcache_invalidate_cache(dataset_alternate, cat=None): + """invalidate GeoWebCache Cache Rules""" + if dataset_alternate is not None and len(dataset_alternate) and "None" not in dataset_alternate: + try: + if cat is None or cat.get_layer(dataset_alternate) is not None: + url = settings.OGC_SERVER["default"]["LOCATION"] + user = settings.OGC_SERVER["default"]["USER"] + passwd = settings.OGC_SERVER["default"]["PASSWORD"] + """ + curl -v -u admin:geoserver \ + -H "Content-type: text/xml" \ + -d "{dataset_alternate}" \ + http://localhost:8080/geoserver/gwc/rest/masstruncate + """ + headers = {"Content-type": "text/xml"} + payload = f"{dataset_alternate}" + r = requests.post( + f"{url.rstrip('/')}/gwc/rest/masstruncate", + headers=headers, + data=payload, + auth=HTTPBasicAuth(user, passwd), + ) + if r.status_code < 200 or r.status_code > 201: + logger.debug(f"Could not Truncate GWC Cache for Dataset '{dataset_alternate}'.") + except Exception: + tb = traceback.format_exc() + logger.debug(tb) + + +def allow_layer_to_all(instance): + """assign access permissions to all users + + This method is only relevant to Dataset instances that have their + underlying data managed by geoserver, meaning: + + * layers that are not associated with a Service + * layers that are associated with a Service that is being CASCADED through + geoserver + + """ + resource = instance.get_self_resource() + logger.debug(f"Inside allow_layer_to_all for instance {instance}") + workspace = get_dataset_workspace(resource.dataset) + dataset_name = ( + resource.dataset.name if resource.dataset and hasattr(resource.dataset, "name") else resource.dataset.alternate + ) + logger.debug(f"Allowing {workspace}:{dataset_name} access to everybody") + try: + priority = gf_utils.get_first_available_priority() + geofence.insert_rule(Rule(Rule.ALLOW, priority=priority, workspace=workspace, layer=dataset_name)) + except Exception as e: + tb = traceback.format_exc() + logger.debug(tb) + raise RuntimeError(f"Could not ADD GeoServer ANONYMOUS Rule for Dataset {dataset_name}: {e}") + finally: + if not getattr(settings, "DELAYED_SECURITY_SIGNALS", False): + invalidate_geofence_cache() + else: + resource.set_dirty_state() + + +# def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_perms=None): +def create_geofence_rules(layer, perms, user=None, group=None, batch: Batch = None): + """ + Collect GeoFence rules related to passed perms into a Batch. + If the batch does not exist, it is created and returned. + """ + if user and group: + raise ValueError("Both user and group given") + + layer_name = layer.name if layer and hasattr(layer, "name") else layer.alternate + workspace_name = get_dataset_workspace(layer) + # Create new rule-set + gf_services = _get_gf_services(layer, perms) + + username = (user if isinstance(user, str) else user.username) if user else None + groupname = (group if isinstance(group, str) else group.name) if group else None + + users_geolimits, groups_geolimits, anonymous_geolimits = get_geolimits(layer, username, groupname) + + if not batch: + batch = AutoPriorityBatch(gf_utils.get_first_available_priority(), f"Sync {workspace_name}:{layer_name}") + + # Set global geolimits + # Anon limits should go at the end, but it's responsibility of the caller to create first user/group rules + for limits, scope, u, g in ( + (users_geolimits, "USER", username, None), + (groups_geolimits, "GROUP", None, groupname), + (anonymous_geolimits, "ANON", None, None), + ): + if limits and limits.exists(): + logger.debug(f"Adding GeoFence {scope} GeoLimit rule: U:{u} G:{g} L:{layer} ") + wkt = limits.last().wkt + batch.add_insert_rule( + Rule( + Rule.LIMIT, + workspace=workspace_name, + layer=layer_name, + user=u, + group=g, + geo_limit=wkt, + catalog_mode=Rule.CM_MIXED, + ) + ) + + if username: + msg = "Adding GeoFence USER rules: U:{username} S:{service} L:{layer} " + elif not groupname: + msg = "Adding GeoFence ANON rules: S:{service} L:{layer} " + if groupname: + msg = "Adding GeoFence GROUP rules: G:{groupname} S:{service} L:{layer} " + + # Set services rules + for rule_fields in gf_services: + if layer and layer_name: + logger.debug(msg.format(username=username, groupname=groupname, layer=layer_name, **rule_fields)) + batch.add_insert_rule( + Rule( + user=username, + group=groupname, + workspace=workspace_name, + layer=layer_name, + **rule_fields, # access, service, request, subfield + ) + ) + + return batch + + +def sync_resources_with_guardian(resource=None, force=False): + """ + Sync resources with Guardian and clear their dirty state + """ + from geonode.layers.models import Dataset + + if resource: + datasets = Dataset.objects.filter(id=resource.id) + else: + if force: + datasets = Dataset.objects.all() + else: + datasets = Dataset.objects.filter(dirty_state=True) + if datasets and datasets.exists(): + logger.debug(" --------------------------- synching with guardian!") + + rules_committed = False + + for dataset in datasets: + try: + batch = AutoPriorityBatch(gf_utils.get_first_available_priority(), f"Sync resources {dataset}") + + gf_utils.collect_delete_layer_rules(get_dataset_workspace(dataset), dataset.name, batch) + + perm_spec = dataset.get_all_level_info() + # All the other users + if "users" in perm_spec: + for user, perms in perm_spec["users"].items(): + user = get_user_model().objects.get(username=user) + # Set the GeoFence User Rules + geofence_user = str(user) + if "AnonymousUser" in geofence_user or str(get_anonymous_user()) in geofence_user: + geofence_user = None + create_geofence_rules(dataset, perms, user=geofence_user, batch=batch) + # All the other groups + if "groups" in perm_spec: + for group, perms in perm_spec["groups"].items(): + group = Group.objects.get(name=group) + if group and group.name and group.name == "anonymous": + group = None + # Set the GeoFence Group Rules + create_geofence_rules(dataset, perms, group=group, batch=batch) + + logger.info(f"Going to synch permissions in GeoFence for resource {dataset}") + rules_committed = geofence.run_batch(batch) + dataset.clear_dirty_state() + except Exception as e: + logger.exception(e) + logger.warning(f"!WARNING! - Failure Synching-up Security Rules for Resource [{dataset}]") + + if rules_committed: + invalidate_geofence_cache() + + +def get_geolimits(layer, username, groupname): + users_geolimits = None + groups_geolimits = None + anonymous_geolimits = None + + if username: + users_geolimits = layer.users_geolimits.filter(user=get_user_model().objects.get(username=username)) + + if groupname: + if GroupProfile.objects.filter(group__name=groupname).exists(): + groups_geolimits = layer.groups_geolimits.filter(group=GroupProfile.objects.get(group__name=groupname)) + + if not username and not groupname: + anonymous_geolimits = layer.users_geolimits.filter(user=get_anonymous_user()) + + return users_geolimits, groups_geolimits, anonymous_geolimits + + +def has_geolimits(layer, user, group): + if user: + _user = user if isinstance(user, str) else user.username + users_geolimits = layer.users_geolimits.filter(user=get_user_model().objects.get(username=_user)) + return users_geolimits.exists() + + if group: + _group = group if isinstance(group, str) else group.name + if GroupProfile.objects.filter(group__name=_group).count() == 1: + groups_geolimits = layer.groups_geolimits.filter(group=GroupProfile.objects.get(group__name=_group)) + return groups_geolimits.exists() + + if not user and not group: + anonymous_geolimits = layer.users_geolimits.filter(user=get_anonymous_user()) + return anonymous_geolimits.exists() + + +def _get_gf_services(layer, perms): + edit = "change_dataset_data" in perms + download = "download_resourcebase" in perms + view = "view_resourcebase" in perms + + gf_services = [] + + # view services + if view or "change_dataset_style" in perms: + gf_services.append({"service": "WMS", "access": True}) + gf_services.append({"service": "GWC", "access": True}) + + # WPS + if view or download or edit: + if not download: + if geoserver_allows_wps_rules(gs_catalog.get_version()): + gf_services.append({"service": "WPS", "subfield": "GS:DOWNLOAD", "access": False}) + gf_services.append({"service": "WPS", "access": True}) + + if download and not edit and layer.is_vector(): + for request in ("TRANSACTION", "LOCKFEATURE", "GETFEATUREWITHLOCK"): + gf_services.append({"service": "WFS", "request": request, "access": False}) + + if download or edit: + service = "WFS" if layer.is_vector() else "WCS" + gf_services.append({"service": service, "access": True}) + + # TODO: check if this rule is really needed + if download and (view or edit): + gf_services.append({"service": "*", "access": True}) + + return gf_services + + +def geoserver_allows_wps_rules(ver: str) -> bool: + try: + s = re.search(r"^[0-9]\.[0-9]*", ver) + if s: + ver_clean = s.group() + return version.parse(ver_clean) >= version.parse("2.23.0") + else: + logger.warning(f'Unparsable GeoServer version string "{ver}"') + + except Exception as e: + logger.warning(f'Error evaluating GeoServer version "{ver}": {e}') + + return False + + +def _get_gwc_filters_and_formats(disable_cache: bool) -> typing.Tuple[list, list]: + filters = [{"styleParameterFilter": {"STYLES": ""}}] + formats = [ + "application/json;type=utfgrid", + "image/gif", + "image/jpeg", + "image/png", + "image/png8", + "image/vnd.jpeg-png", + "image/vnd.jpeg-png8", + ] + if disable_cache: + filters = None + formats = None + return filters, formats diff --git a/geonode/geoserver/signals.py b/geonode/geoserver/signals.py new file mode 100644 index 0000000..0f3a847 --- /dev/null +++ b/geonode/geoserver/signals.py @@ -0,0 +1,161 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import errno +import logging +from requests.exceptions import ConnectionError + +from deprecated import deprecated +from geoserver.layer import Layer as GsLayer + +from django.db.models import Q +from django.dispatch import Signal + +# use different name to avoid module clash +from geonode.utils import is_monochromatic_image +from geonode.decorators import on_ogc_backend +from geonode.geoserver.helpers import gs_catalog, ogc_server_settings +from geonode.geoserver.tasks import geoserver_create_thumbnail +from geonode.layers.models import Dataset +from geonode.services.enumerations import CASCADED + +from . import BACKEND_PACKAGE +from .tasks import geoserver_cascading_delete, geoserver_post_save_datasets + +logger = logging.getLogger("geonode.geoserver.signals") + +geoserver_automatic_default_style_set = Signal() + +geofence_rule_assign = Signal() + + +def geoserver_delete(typename): + # cascading_delete should only be called if + # ogc_server_settings.BACKEND_WRITE_ENABLED == True + if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): + geoserver_cascading_delete.apply_async(args=(typename,), expiration=30) + + +@on_ogc_backend(BACKEND_PACKAGE) +def geoserver_pre_delete(instance, sender, **kwargs): + """Removes the layer from GeoServer""" + # cascading_delete should only be called if + # ogc_server_settings.BACKEND_WRITE_ENABLED == True + if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): + if ( + not hasattr(instance, "remote_service") + or instance.remote_service is None + or instance.remote_service.method == CASCADED + ): + if instance.alternate: + geoserver_cascading_delete.apply_async(args=(instance.alternate,), expiration=30) + + +@on_ogc_backend(BACKEND_PACKAGE) +def geoserver_post_save_local(instance, *args, **kwargs): + """Send information to geoserver. + + The attributes sent include: + + * Title + * Abstract + * Name + * Keywords + * Metadata Links, + * Point of Contact name and url + """ + geoserver_post_save_datasets.apply_async(args=(instance.id, args, kwargs), expiration=30) + + +@on_ogc_backend(BACKEND_PACKAGE) +def geoserver_pre_save_maplayer(instance, sender, **kwargs): + # If this object was saved via fixtures, + # do not do post processing. + if kwargs.get("raw", False): + return + + try: + instance.local = isinstance(gs_catalog.get_layer(instance.name), GsLayer) + except ConnectionError as e: + logger.warning(f"Could not connect to catalog to verify if layer {instance.name} was local: {e}") + except OSError as e: + logger.warning(f"***** OSERROR TYPE:{type(e)} ERR:{e} ERRNO:{e.errno}") + if e.errno == errno.ECONNREFUSED: + msg = f"Could not connect to catalog to verify if layer {instance.name} was local" + logger.warning(msg) + else: + raise e + + # Set dataset + if instance.dataset is None: + dataset_queryset = Dataset.objects.filter(Q(alternate=instance.name) | Q(name=instance.name)) + if instance.local and instance.store: + dataset_queryset = dataset_queryset.filter(store=instance.store) + elif instance.ows_url: + dataset_queryset = dataset_queryset.filter(remote_service__base_url=instance.ows_url) + try: + instance.dataset = dataset_queryset.get() + except (Dataset.DoesNotExist, Dataset.MultipleObjectsReturned): + pass + + +@deprecated(version="3.2.1", reason="Use direct calls to the ReourceManager.") +def geoserver_post_save_map(instance, sender, created, **kwargs): + instance.set_missing_info() + if not created: + if not instance.thumbnail_url: + logger.debug(f"... Creating Thumbnail for Map [{instance.title}]") + geoserver_create_thumbnail.apply_async( + args=( + instance.id, + False, + True, + ), + expiration=30, + ) + + +@deprecated(version="3.2.1", reason="Use direct calls to the ReourceManager.") +def geoserver_set_thumbnail(instance, **kwargs): + # Creating Dataset Thumbnail + # some thumbnail generators will update thumbnail_url. If so, don't + # immediately re-generate the thumbnail here. use layer#save(update_fields=['thumbnail_url']) + try: + logger.debug(f"... Creating Thumbnail for Dataset {instance.title}") + _recreate_thumbnail = False + if ( + "update_fields" in kwargs + and kwargs["update_fields"] is not None + and "thumbnail_url" in kwargs["update_fields"] + ): + _recreate_thumbnail = True + if not instance.thumbnail_url or is_monochromatic_image(instance.thumbnail_url): + _recreate_thumbnail = True + if _recreate_thumbnail: + geoserver_create_thumbnail.apply_async( + args=( + instance.id, + False, + True, + ), + expiration=30, + ) + else: + logger.debug(f"... Thumbnail for Dataset {instance.title} already exists: {instance.thumbnail_url}") + except Exception as e: + logger.exception(e) diff --git a/geonode/geoserver/tasks.py b/geonode/geoserver/tasks.py new file mode 100644 index 0000000..6cd6a4a --- /dev/null +++ b/geonode/geoserver/tasks.py @@ -0,0 +1,371 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +import os + +from django.conf import settings + +from celery import shared_task +from celery.utils.log import get_task_logger + +from geonode.celery_app import app +from geonode.tasks.tasks import AcquireLock, FaultTolerantTask +from geonode.base.models import Link +from geonode.base import enumerations +from geonode.layers.models import Dataset +from geonode.base.models import ResourceBase + +from .security import sync_resources_with_guardian +from .helpers import ( + gs_slurp, + gs_catalog, + set_styles, + get_sld_for, + set_dataset_style, + cascading_delete, + create_gs_thumbnail, + sync_instance_with_geoserver, +) + +logger = get_task_logger(__name__) + +log_lock = logging.getLogger("geonode_lock_handler") + + +@app.task( + bind=True, + base=FaultTolerantTask, + name="geonode.geoserver.tasks.geoserver_update_datasets", + queue="geoserver.catalog", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def geoserver_update_datasets(self, *args, **kwargs): + """ + Runs update layers. + """ + lock_id = f"{self.request.id}" + log_lock.debug(f"geoserver_update_datasets: Creating lock {lock_id}") + with AcquireLock(lock_id) as lock: + log_lock.debug(f"geoserver_update_datasets: Acquiring lock {lock_id}") + if lock.acquire() is True: + log_lock.debug(f"geoserver_update_datasets: Acquired lock {lock_id}") + try: + return gs_slurp(*args, **kwargs) + finally: + lock.release() + log_lock.debug(f"geoserver_update_datasets: Released lock {lock_id}") + + +@app.task( + bind=True, + base=FaultTolerantTask, + name="geonode.geoserver.tasks.geoserver_set_style", + queue="geoserver.catalog", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def geoserver_set_style(self, instance_id, base_file): + """ + Sets styles from SLD file. + """ + instance = None + try: + instance = Dataset.objects.get(id=instance_id) + except Dataset.DoesNotExist: + logger.debug(f"Dataset id {instance_id} does not exist yet!") + raise + + lock_id = f"{self.request.id}" if self.request.id else instance.name + log_lock.debug(f"geoserver_set_style: Creating lock {lock_id} for {instance.name}") + with AcquireLock(lock_id) as lock: + log_lock.debug(f"geoserver_set_style: Acquiring lock {lock_id} for {instance.name}") + if lock.acquire() is True: + log_lock.debug(f"geoserver_set_style: Acquired lock {lock_id} for {instance.name}") + try: + sld = None + if os.path.exists(base_file): + sld = open(base_file, "rb").read() + else: + sld = base_file + base_file = None + + set_dataset_style(instance, instance.alternate, sld, base_file=base_file) + + except Exception as e: + logger.exception(e) + finally: + lock.release() + log_lock.debug(f"geoserver_set_style: Released lock {lock_id} for {instance.name}") + + +@app.task( + bind=True, + base=FaultTolerantTask, + name="geonode.geoserver.tasks.geoserver_create_style", + queue="geoserver.catalog", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def geoserver_create_style(self, instance_id, name, sld_file, tempdir): + """ + Sets or create styles from Upload Session. + """ + from geonode.geoserver.signals import geoserver_automatic_default_style_set + + instance = None + try: + instance = Dataset.objects.get(id=instance_id) + except Dataset.DoesNotExist: + logger.debug(f"Dataset id {instance_id} does not exist yet!") + raise + + lock_id = f"{self.request.id}" if self.request.id else instance.name + log_lock.debug(f"geoserver_create_style: Creating lock {lock_id} for {instance.name}") + with AcquireLock(lock_id) as lock: + log_lock.debug(f"geoserver_create_style: Acquiring lock {lock_id} for {instance.name}") + if lock.acquire() is True: + log_lock.debug(f"geoserver_create_style: Acquired lock {lock_id} for {instance.name}") + try: + f = None + if sld_file and os.path.exists(sld_file) and os.access(sld_file, os.R_OK): + if os.path.isfile(sld_file): + try: + f = open(sld_file) + except Exception: + pass + elif tempdir and os.path.exists(tempdir): + if os.path.isfile(os.path.join(tempdir, sld_file)): + try: + f = open(os.path.join(tempdir, sld_file)) + except Exception: + pass + if f: + sld = f.read() + f.close() + if not gs_catalog.get_style(name=name, workspace=settings.DEFAULT_WORKSPACE): + style = gs_catalog.create_style(name, sld, raw=True, workspace=settings.DEFAULT_WORKSPACE) + gs_dataset = gs_catalog.get_layer(name) + _default_style = gs_dataset.default_style + gs_dataset.default_style = style + gs_catalog.save(gs_dataset) + set_styles(instance, gs_catalog) + try: + gs_catalog.delete(_default_style) + Link.objects.filter( + resource=instance.resourcebase_ptr, + name="Legend", + url__contains=f"STYLE={_default_style.name}", + ).delete() + except Exception as e: + logger.exception(e) + else: + get_sld_for(gs_catalog, instance) + else: + get_sld_for(gs_catalog, instance) + if not f: + # this signal is used by the mapstore client to set the style in visual mode + geoserver_automatic_default_style_set.send_robust(sender=instance, instance=instance) + finally: + lock.release() + log_lock.debug(f"geoserver_create_style: Released lock {lock_id} for {instance.name}") + + +@app.task( + bind=True, + base=FaultTolerantTask, + name="geonode.geoserver.tasks.geoserver_post_save_datasets", + queue="geoserver.catalog", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def geoserver_post_save_datasets(self, instance_id, *args, **kwargs): + """ + Runs update layers. + """ + instance = None + try: + instance = Dataset.objects.get(id=instance_id) + except Dataset.DoesNotExist: + logger.debug(f"Dataset id {instance_id} does not exist yet!") + raise + + lock_id = f"{self.request.id}" if self.request.id else instance.name + log_lock.debug(f"geoserver_post_save_datasets: Creating lock {lock_id} for {instance_id}") + with AcquireLock(lock_id) as lock: + log_lock.debug(f"geoserver_post_save_datasets: Acquiring lock {lock_id} for {instance_id}") + if lock.acquire() is True: + log_lock.debug(f"geoserver_post_save_datasets: Acquired lock {lock_id} for {instance_id}") + try: + sync_instance_with_geoserver(instance_id, *args, **kwargs) + finally: + lock.release() + log_lock.debug(f"geoserver_post_save_datasets: Releasing lock {lock_id} for {instance_id}") + + +@app.task( + bind=True, + base=FaultTolerantTask, + name="geonode.geoserver.tasks.geoserver_create_thumbnail", + queue="geoserver.events", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def geoserver_create_thumbnail(self, instance_id, overwrite=True, check_bbox=True): + """ + Runs create_gs_thumbnail. + """ + instance = None + try: + instance = ResourceBase.objects.get(id=instance_id).get_real_instance() + except Exception: + logger.error(f"Resource id {instance_id} does not exist yet!") + raise + + lock_id = f"{self.request.id}" if self.request.id else instance.name + log_lock.debug(f"geoserver_create_thumbnail: Creating lock {lock_id} for {instance.name}") + with AcquireLock(lock_id) as lock: + log_lock.debug(f"geoserver_create_thumbnail: Acquiring lock {lock_id} for {instance.name}") + if lock.acquire() is True: + log_lock.debug(f"geoserver_create_thumbnail: Acquired lock {lock_id} for {instance.name}") + try: + instance.set_processing_state(enumerations.STATE_RUNNING) + try: + instance.set_dirty_state() + create_gs_thumbnail(instance, overwrite=overwrite, check_bbox=check_bbox) + logger.debug(f"... Created Thumbnail for Dataset {instance.title}") + except Exception as e: + geoserver_create_thumbnail.retry(exc=e) + finally: + instance.set_processing_state(enumerations.STATE_PROCESSED) + finally: + lock.release() + log_lock.debug(f"geoserver_create_thumbnail: Released lock {lock_id} for {instance.name}") + + +@app.task( + bind=True, + base=FaultTolerantTask, + name="geonode.geoserver.tasks.geoserver_cascading_delete", + queue="cleanup", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def geoserver_cascading_delete(self, *args, **kwargs): + """ + Runs cascading_delete. + """ + lock_id = f"{self.request.id}" + with AcquireLock(lock_id) as lock: + if lock.acquire() is True: + try: + return cascading_delete(*args, **kwargs) + finally: + lock.release() + + +@app.task( + bind=True, + name="geonode.geoserver.tasks.geoserver_delete_map", + queue="cleanup", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def geoserver_delete_map(self, object_id): + """ + Deletes a map and the associated map layers. + """ + from geonode.maps.models import Map + + lock_id = f"{self.request.id}" + with AcquireLock(lock_id) as lock: + if lock.acquire() is True: + try: + try: + map_obj = Map.objects.get(id=object_id) + except Map.DoesNotExist: + return + + map_obj.maplayers.all().delete() + map_obj.delete() + finally: + lock.release() + + +@shared_task( + bind=True, + name="geonode.security.tasks.synch_guardian", + queue="security", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def synch_guardian(): + """ + Sync resources with Guardian and clear their dirty state + """ + if getattr(settings, "DELAYED_SECURITY_SIGNALS", False): + sync_resources_with_guardian() diff --git a/geonode/geoserver/templates/geoserver/dataset_download.xml b/geonode/geoserver/templates/geoserver/dataset_download.xml new file mode 100644 index 0000000..fbdf348 --- /dev/null +++ b/geonode/geoserver/templates/geoserver/dataset_download.xml @@ -0,0 +1,29 @@ + + + gs:Download + + + layerName + + {{alternate}} + + + + outputFormat + + {{download_format}} + + + + cropToROI + + false + + + + + + result + + + \ No newline at end of file diff --git a/geonode/geoserver/templates/geoserver/layer.xml b/geonode/geoserver/templates/geoserver/layer.xml new file mode 100644 index 0000000..1137135 --- /dev/null +++ b/geonode/geoserver/templates/geoserver/layer.xml @@ -0,0 +1,35 @@ + +{{ layer.name }} +{{ layer.title }} +{{ layer.abstract }} + + {% for kw in layer.keyword_list %} + {{ kw }} + {% endfor %} + +{{ layer.srid }} + + {{ layer.bbox_x0 }} + {{ layer.bbox_x1 }} + {{layer.bbox_y0 }} + {{ layer.bbox_y1 }} + + + +{{ layer.owner.username }} + + +text/xml + + +{% for style in layer.styles.all %} + +{% endfor %} + diff --git a/geonode/geoserver/tests/__init__.py b/geonode/geoserver/tests/__init__.py new file mode 100644 index 0000000..93c32d9 --- /dev/null +++ b/geonode/geoserver/tests/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/geoserver/tests/integration.py b/geonode/geoserver/tests/integration.py new file mode 100644 index 0000000..39d8040 --- /dev/null +++ b/geonode/geoserver/tests/integration.py @@ -0,0 +1,354 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this profgram. If not, see . +# +######################################################################### + +from geonode.layers.populate_datasets_data import create_dataset_data +from geonode.geoserver.createlayer.utils import create_dataset +from geonode.tests.base import GeoNodeLiveTestSupport + +import timeout_decorator + +import time +import logging +from lxml import etree +from owslib.etree import etree as dlxml +from urllib.request import urlopen, Request +from urllib.parse import urljoin + +from django.conf import settings +from django.urls import reverse +from django.contrib.auth import get_user_model +from django.test.utils import override_settings + +from geonode import geoserver +from geonode.layers.models import Dataset +from geonode.compat import ensure_string +from geonode.tests.utils import check_dataset +from geonode.decorators import on_ogc_backend +from geonode.base.models import TopicCategory, Link +from geonode.geoserver.helpers import set_attributes_from_geoserver + +LOCAL_TIMEOUT = 300 + +LOGIN_URL = "/accounts/login/" + +logger = logging.getLogger(__name__) + + +def _log(msg, *args): + logger.debug(msg, *args) + + +@override_settings(SITEURL="http://localhost:8001/") +class GeoNodeGeoServerSync(GeoNodeLiveTestSupport): + """ + Tests GeoNode/GeoServer syncronization + """ + + port = 8001 + + def setUp(self): + super(GeoNodeLiveTestSupport, self).setUp() + settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] = True + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @timeout_decorator.timeout(LOCAL_TIMEOUT) + def test_set_attributes_from_geoserver(self): + """Test attributes syncronization""" + layer = Dataset.objects.all().first() + create_dataset_data(layer.resourcebase_ptr_id) + try: + # set attributes for resource + for attribute in layer.attribute_set.all(): + attribute.attribute_label = f"{attribute.attribute}_label" + attribute.description = f"{attribute.attribute}_description" + attribute.save() + + # tests if everything is synced properly + for attribute in layer.attribute_set.all(): + self.assertEqual(attribute.attribute_label, f"{attribute.attribute}_label") + self.assertEqual(attribute.description, f"{attribute.attribute}_description") + + # sync the attributes with GeoServer + # since on geoserver are empty, we expect that now the layer + # does not have any attribute + set_attributes_from_geoserver(layer) + + links = Link.objects.filter(resource=layer.resourcebase_ptr) + self.assertIsNotNone(links) + self.assertTrue(len(links) >= 7) + + original_data_links = [ll for ll in links if "original" == ll.link_type] + self.assertEqual(len(original_data_links), 0) + + finally: + # Clean up and completely delete the layers + layer.delete() + + +@override_settings(SITEURL="http://localhost:8002/") +class GeoNodeGeoServerCapabilities(GeoNodeLiveTestSupport): + """ + Tests GeoNode/GeoServer GetCapabilities per layer, user, category and map + """ + + port = 8002 + + def setUp(self): + super(GeoNodeLiveTestSupport, self).setUp() + settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] = True + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @timeout_decorator.timeout(LOCAL_TIMEOUT) + def test_capabilities(self): + """Test capabilities""" + + # a category + category = TopicCategory.objects.first() + + # some users + norman = get_user_model().objects.get(username="norman") + admin = get_user_model().objects.get(username="admin") + + # create 3 layers, 2 with norman as an owner an 2 with category as a category + layer1 = create_dataset( + name="layer1", title="san_andres_y_providencia_poi", owner_name=norman, geometry_type="Point" + ) + layer2 = create_dataset(name="layer2", title="single_point", owner_name=norman, geometry_type="Point") + layer2.category = category + layer2.save() + layer3 = create_dataset( + name="layer3", title="san_andres_y_providencia_administrative", owner_name=admin, geometry_type="Point" + ) + layer3.category = category + layer3.save() + try: + namespaces = { + "wms": "http://www.opengis.net/wms", + "xlink": "http://www.w3.org/1999/xlink", + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + } + + # 0. test capabilities_dataset + url = reverse("capabilities_dataset", args=[layer1.id]) + resp = self.client.get(url) + layercap = dlxml.fromstring(resp.content) + rootdoc = etree.ElementTree(layercap) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) + layernode = layernodes[0] + + self.assertEqual(1, len(layernodes)) + self.assertEqual(layernode.find("wms:Name", namespaces).text, layer1.name) + + # 1. test capabilities_user + url = reverse("capabilities_user", args=[norman.username]) + resp = self.client.get(url) + layercap = dlxml.fromstring(resp.content) + rootdoc = etree.ElementTree(layercap) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) + + # norman has 2 layers + self.assertEqual(1, len(layernodes)) + + # the norman two layers are named layer1 and layer2 + count = 0 + for layernode in layernodes: + if layernode.find("wms:Name", namespaces).text == layer1.name: + count += 1 + elif layernode.find("wms:Name", namespaces).text == layer2.name: + count += 1 + self.assertEqual(1, count) + + # 2. test capabilities_category + url = reverse("capabilities_category", args=[category.identifier]) + resp = self.client.get(url) + layercap = dlxml.fromstring(resp.content) + rootdoc = etree.ElementTree(layercap) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) + + # category is in two layers + self.assertEqual(1, len(layernodes)) + + # the layers for category are named layer1 and layer3 + count = 0 + for layernode in layernodes: + if layernode.find("wms:Name", namespaces).text == layer1.name: + count += 1 + elif layernode.find("wms:Name", namespaces).text == layer3.name: + count += 1 + self.assertEqual(1, count) + + # 3. test for a map + # TODO + finally: + # Clean up and completely delete the layers + layer1.delete() + layer2.delete() + layer3.delete() + + +@override_settings(SITEURL="http://localhost:8003/") +class GeoNodePermissionsTest(GeoNodeLiveTestSupport): + """ + Tests GeoNode permissions and its integration with GeoServer + """ + + port = 8003 + + def setUp(self): + super(GeoNodeLiveTestSupport, self).setUp() + settings.OGC_SERVER["default"]["GEOFENCE_SECURITY_ENABLED"] = True + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @timeout_decorator.timeout(LOCAL_TIMEOUT) + def test_unpublished(self): + """Test permissions on an unpublished layer""" + layer = Dataset.objects.first() + layer.set_default_permissions() + check_dataset(layer) + + # we need some time to have the service up and running + # time.sleep(20) + + try: + # request getCapabilities: layer must be there as it is published and + # advertised: we need to check if in response there is + # geonode:san_andres_y_providencia_water + geoserver_base_url = settings.OGC_SERVER["default"]["LOCATION"] + get_capabilities_url = "ows?" "service=wms&version=1.3.0&request=GetCapabilities" + url = urljoin(geoserver_base_url, get_capabilities_url) + str_to_check = f"geonode:{layer.name}" + request = Request(url) + response = urlopen(request) + + # by default the uploaded layer is published + self.assertTrue(layer.is_published) + self.assertTrue(any(str_to_check in ensure_string(s) for s in response.readlines())) + finally: + # Clean up and completely delete the layer + layer.delete() + + # with settings disabled + with self.settings(RESOURCE_PUBLISHING=True): + layer = Dataset.objects.first() + layer.is_approved = False + layer.is_published = False + layer.save() + layer.set_default_permissions() + check_dataset(layer) + + # we need some time to have the service up and running + time.sleep(20) + + try: + # by default the uploaded layer must be unpublished + self.assertEqual(layer.is_published, False) + + # check the layer is not in GetCapabilities + request = Request(url) + response = urlopen(request) + + # now test with published layer + layer = Dataset.objects.get(pk=layer.pk) + layer.is_published = True + layer.save() + + # we need some time to have the service up and running + time.sleep(20) + + request = Request(url) + response = urlopen(request) + self.assertTrue(any(str_to_check in ensure_string(s) for s in response.readlines())) + finally: + # Clean up and completely delete the layer + layer.delete() + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @timeout_decorator.timeout(LOCAL_TIMEOUT) + def test_default_anonymous_permissions(self): + anonymous = get_user_model().objects.get(username="AnonymousUser") + norman = get_user_model().objects.get(username="norman") + with override_settings( + RESOURCE_PUBLISHING=False, + ADMIN_MODERATE_UPLOADS=False, + DEFAULT_ANONYMOUS_VIEW_PERMISSION=True, + DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=False, + ): + self.client.login(username="norman", password="norman") + + saved_dataset = create_dataset( + name="san_andres_y_providencia_poi_by_norman", + title="san_andres_y_providencia_poi", + owner_name=norman, + geometry_type="Point", + ) + + try: + namespaces = { + "wms": "http://www.opengis.net/wms", + "xlink": "http://www.w3.org/1999/xlink", + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + } + url = urljoin(settings.SITEURL, reverse("capabilities_dataset", args=[saved_dataset.id])) + resp = self.client.get(url) + content = resp.content + self.assertTrue(content) + layercap = dlxml.fromstring(content) + rootdoc = etree.ElementTree(layercap) + layernodes = rootdoc.findall("./[wms:Name]", namespaces) + layernode = layernodes[0] + self.assertEqual(1, len(layernodes)) + self.assertEqual(layernode.find("wms:Name", namespaces).text, saved_dataset.name) + self.client.logout() + resp = self.client.get(url) + layercap = dlxml.fromstring(resp.content) + self.assertIsNotNone(layercap) + finally: + # annonymous can not download created resource + self.assertEqual(["view_resourcebase"], self.get_user_resource_perms(saved_dataset, anonymous)) + # Cleanup + saved_dataset.delete() + + with override_settings(DEFAULT_ANONYMOUS_VIEW_PERMISSION=False, DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True): + saved_dataset = create_dataset( + name="san_andres_y_providencia_poi_by_norman", + title="san_andres_y_providencia_poi", + owner_name=norman, + geometry_type="Point", + ) + # annonymous can view/download created resource + self.assertEqual( + ["download_resourcebase", "view_resourcebase"], self.get_user_resource_perms(saved_dataset, anonymous) + ) + # Cleanup + saved_dataset.delete() + + with override_settings(DEFAULT_ANONYMOUS_VIEW_PERMISSION=False, DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=False): + saved_dataset = create_dataset( + name="san_andres_y_providencia_poi_by_norman", + title="san_andres_y_providencia_poi", + owner_name=norman, + geometry_type="Point", + ) + # annonymous can not view/download created resource + self.assertEqual([], self.get_user_resource_perms(saved_dataset, anonymous)) + # Cleanup + saved_dataset.delete() + + def get_user_resource_perms(self, instance, user): + return list(instance.get_user_perms(user).union(instance.get_self_resource().get_user_perms(user))) diff --git a/geonode/geoserver/tests/test_helpers.py b/geonode/geoserver/tests/test_helpers.py new file mode 100644 index 0000000..2fb1b3c --- /dev/null +++ b/geonode/geoserver/tests/test_helpers.py @@ -0,0 +1,277 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import re +import time +import logging + +from urllib.parse import urljoin + +from django.conf import settings +from django.urls import reverse + +from geonode import geoserver +from geonode.decorators import on_ogc_backend +from geonode.base.populate_test_data import all_public, create_models, remove_models +from geonode.layers.populate_datasets_data import create_dataset_data +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.geoserver.views import _response_callback +from geonode.geoserver.helpers import ( + gs_catalog, + ows_endpoint_in_path, + get_dataset_storetype, + extract_name_from_sld, + get_dataset_capabilities_url, + get_layer_ows_url, +) +from geonode.geoserver.ows import _wcs_link, _wfs_link, _wms_link + + +logger = logging.getLogger(__name__) + + +class HelperTest(GeoNodeBaseTestSupport): + type = "dataset" + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.user = "admin" + self.passwd = "admin" + create_dataset_data() + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_extract_name_from_sld(self): + content = """ + ]> + + + + &ent; + + +""" + self.assertIsNone(extract_name_from_sld(gs_catalog, content)) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_replace_callback(self): + content = f""" + GeoNode Local GeoServer + This is a description of your Web Map Server. + + EPSG:4326 + EPSG:3785 + EPSG:3857 + EPSG:900913 + EPSG:32647 + EPSG:32736 + CRS:84 + + -124.731422 + 12.512771464573753 + 12.4801497 + 49.371735 + + + + + + + + + + geonode:DE_USNG_UTM18 + DE_USNG_UTM18 + No abstract provided + + DE_USNG_UTM18 + features + + EPSG:26918 + CRS:84 + + -75.93570725669369 + -75.00000000000001 + 38.3856300861002 + 39.89406880610797 + + + + + + + + + + + other + + + + other + + + + other + + + + other + + + + text/xml + + + + other + + + + other + + + + """ + kwargs = {"content": content, "status": 200, "content_type": "application/xml"} + _content = _response_callback(**kwargs).content + self.assertTrue(re.findall(f'{urljoin(settings.SITEURL, "/gs/")}ows', str(_content))) + + kwargs = {"content": content, "status": 200, "content_type": "text/xml; charset=UTF-8"} + _content = _response_callback(**kwargs).content + self.assertTrue(re.findall(f'{urljoin(settings.SITEURL, "/gs/")}ows', str(_content))) + + def test_return_element_if_not_exists_in_the_subtypes(self): + el = get_dataset_storetype("not-existing-type") + self.assertEqual("not-existing-type", el) + + def test_datastore_should_return_vector(self): + el = get_dataset_storetype("dataStore") + self.assertEqual("vector", el) + + def test_coverageStore_should_return_raster(self): + el = get_dataset_storetype("coverageStore") + self.assertEqual("raster", el) + + def test_remoteStore_should_return_remote(self): + el = get_dataset_storetype("remoteStore") + self.assertEqual("remote", el) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_geoserver_proxy_strip_paths(self): + response = self.client.get( + f"{reverse('ows_endpoint')}?service=WFS&version=1.1.0&request=DescribeFeatureType&typeName=geonode:tipi_forestali&outputFormat=application/json&access_token=something" + ) + self.assertEqual(response.status_code, 200) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_ows_links(self): + ows_url = "http://foo.org/ows" + identifier = "foo:fake_alternate" + min_x, min_y, max_x, max_y = -1, -1, 1, 1 + expected_url = f"{ows_url}?service=WCS&request=GetCoverage&coverageid=foo__fake_alternate&format=image%2Ftiff&version=2.0.1&compression=DEFLATE&tileWidth=512&tileHeight=512&outputCrs=4326" + download_url = _wcs_link( + ows_url, + identifier, + "image/tiff", + srid="4326", + bbox=[min_x, min_y, max_x, max_y], + compression="DEFLATE", + tile_size=512, + ) + self.assertEqual(download_url, expected_url, download_url) + + expected_url = f"{ows_url}?service=WFS&version=1.0.0&request=GetFeature&typename=foo%3Afake_alternate&outputFormat=application%2Fzip&srs=4326&bbox=%5B-1%2C+-1%2C+1%2C+1%5D" + download_url = _wfs_link( + ows_url, identifier, "application/zip", {}, srid="4326", bbox=[min_x, min_y, max_x, max_y] + ) + self.assertEqual(download_url, expected_url, download_url) + + expected_url = f"{ows_url}?service=WMS&request=GetMap&layers=foo%3Afake_alternate&format=image%2Fpng&height=512&width=512&srs=4326&bbox=%5B-1%2C+-1%2C+1%2C+1%5D" + download_url = _wms_link( + ows_url, identifier, "image/png", 512, 512, srid="4326", bbox=[min_x, min_y, max_x, max_y] + ) + self.assertEqual(download_url, expected_url, download_url) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_ows_endpoint_in_path(self): + path = "http://localhost:8080ows/rest/rest/" + + self.assertIsNotNone(ows_endpoint_in_path(path)) + self.assertEqual(len(ows_endpoint_in_path(path).groups()), 1) + + start_time = time.time() + ows_endpoint_in_path(path) + end_time_1 = time.time() - start_time + + start_time = time.time() + re.match(r".*/(rest)/.*$", path, re.IGNORECASE) + end_time_2 = time.time() - start_time + + self.assertLess(end_time_1, end_time_2) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_dataset_capabilties_url(self): + from geonode.layers.models import Dataset + + ows_url = settings.GEOSERVER_PUBLIC_LOCATION + identifier = "geonode:CA" + dataset = Dataset.objects.get(alternate=identifier) + expected_url = f"{ows_url}geonode/CA/ows?service=wms&version=1.3.0&request=GetCapabilities" + capabilities_url = get_dataset_capabilities_url(dataset) + self.assertEqual(capabilities_url, expected_url, capabilities_url) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_layer_ows_url(self): + from geonode.layers.models import Dataset + + ows_url = settings.GEOSERVER_PUBLIC_LOCATION + identifier = "geonode:CA" + dataset = Dataset.objects.get(alternate=identifier) + expected_url = f"{ows_url}geonode/CA/ows" + capabilities_url = get_layer_ows_url(dataset) + self.assertEqual(capabilities_url, expected_url, capabilities_url) diff --git a/geonode/geoserver/tests/test_manager.py b/geonode/geoserver/tests/test_manager.py new file mode 100644 index 0000000..7d2b9ac --- /dev/null +++ b/geonode/geoserver/tests/test_manager.py @@ -0,0 +1,97 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import base64 +import shutil +from django.test import override_settings +import gisdata +import requests + +from django.conf import settings +from django.contrib.auth import get_user_model + +from geonode import geoserver +from geonode.base import enumerations +from geonode.layers.models import Dataset +from geonode.layers.utils import get_files +from geonode.decorators import on_ogc_backend +from geonode.geoserver.helpers import gs_catalog +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.geoserver.manager import GeoServerResourceManager +from geonode.base.populate_test_data import create_single_dataset + + +class TestGeoServerResourceManager(GeoNodeBaseTestSupport): + def setUp(self): + self.files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + self.files_as_dict, self.tmpdir = get_files(self.files) + self.cat = gs_catalog + self.user = get_user_model().objects.get(username="admin") + self.sut = create_single_dataset("san_andres_y_providencia_water.shp") + self.sut.name = "san_andres_y_providencia_water" + self.sut.save() + self.geoserver_url = settings.GEOSERVER_LOCATION + self.geoserver_manager = GeoServerResourceManager() + + def tearDown(self) -> None: + if self.tmpdir: + shutil.rmtree(self.tmpdir, ignore_errors=True) + return super().tearDown() + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @override_settings(ASYNC_SIGNALS=False, FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777, FILE_UPLOAD_PERMISSIONS=0o7777) + def test_revise_resource_value_in_append_should_add_expected_rows_in_the_catalog(self): + layer = Dataset.objects.get(name=self.sut.name) + gs_layer = self.cat.get_layer("san_andres_y_providencia_water") + if gs_layer is None: + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="create" + ) + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="append" + ) + basic_auth = base64.b64encode(b"admin:geoserver") + result = requests.get( + f"{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}", + headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"}, + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json().get("import").get("state"), enumerations.STATE_COMPLETE) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_revise_resource_value_in_replace_should_add_expected_rows_in_the_catalog(self): + layer = Dataset.objects.get(name=self.sut.name) + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="replace" + ) + basic_auth = base64.b64encode(b"admin:geoserver") + result = requests.get( + f"{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}", + headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"}, + ) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.json().get("import").get("state"), enumerations.STATE_COMPLETE) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_revise_resource_value_in_replace_should_return_none_for_not_existing_dataset(self): + layer = create_single_dataset("fake_dataset") + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="replace" + ) + self.assertEqual(_gs_import_session_info.import_session.state, enumerations.STATE_COMPLETE) diff --git a/geonode/geoserver/tests/test_server.py b/geonode/geoserver/tests/test_server.py new file mode 100644 index 0000000..f43798c --- /dev/null +++ b/geonode/geoserver/tests/test_server.py @@ -0,0 +1,1288 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.tests.base import GeoNodeBaseTestSupport + +import os +import json +import base64 +import shutil + +from os.path import basename, splitext +from urllib.parse import urljoin, urlencode, urlsplit + +from django.conf import settings +from django.urls import reverse +from django.test.client import RequestFactory +from django.contrib.auth import get_user_model +from django.core.management import call_command +from django.test.utils import override_settings +from django.contrib.auth.models import AnonymousUser + +from guardian.shortcuts import assign_perm + +from geonode.geoserver.helpers import ogc_server_settings +from geonode.geoserver.views import check_geoserver_access, style_change_check + +from geonode import geoserver +from geonode.base.models import Configuration +from geonode.decorators import on_ogc_backend + +from geonode.utils import mkdtemp, OGC_Servers_Handler +from geonode.layers.models import Dataset, Style +from geonode.layers.populate_datasets_data import create_dataset_data +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset +from geonode.geoserver.helpers import gs_catalog, get_sld_for, extract_name_from_sld +from geonode.catalogue.models import catalogue_post_save + +import logging + +logger = logging.getLogger(__name__) + +san_andres_y_providencia_sld = """ + + + geonode:san_andres_y_providencia_administrative + + san_andres_y_providencia_administrative + San Andres y Providencia Administrative + 1 + + + + + #880000 + 3 + 4.0 4.0 + + + + + + + + + #ffbbbb + 2 + + + + + + + +""" + +lac_sld = """ + + + LAC_NonIndigenous_Access_to_Sanitation2 + + LAC NonIndigenous Access to Sanitation + + + Low (25 - 40%) + + Low (25 - 40%) + + + + + NonInd + 24.89999999999999858 + + + NonInd + 39.89999999999999858 + + + + + + #ff8b16 + + + #000001 + 0.1 + bevel + + + + + Medium Low (40 - 65 %) + + Medium Low (40 - 65 %) + + + + + NonInd + 39.89999999999999858 + + + NonInd + 64.90000000000000568 + + + + + + #fffb0b + + + #000001 + 0.1 + bevel + + + + + Medium (65 - 70 %) + + Medium (65 - 70 %) + + + + + NonInd + 64.90000000000000568 + + + NonInd + 69.90000000000000568 + + + + + + #55d718 + + + #000001 + 0.1 + bevel + + + + + Medium High (70 - 85 %) + + Medium High (70 - 85 %) + + + + + NonInd + 69.90000000000000568 + + + NonInd + 84.90000000000000568 + + + + + + #3f7122 + + + #000001 + 0.1 + bevel + + + + + High (85 - 93 %) + + High (85 - 93 %) + + + + + NonInd + 84.90000000000000568 + + + NonInd + 92.90000000000000568 + + + + + + #76ffff + + + #000001 + 0.1 + bevel + + + + + Very High (93 - 96 %) + + Very High (93 - 96 %) + + + + + NonInd + 92.90000000000000568 + + + NonInd + 96.09999999999999432 + + + + + + #0a4291 + + + #000001 + 0.1 + bevel + + + + + Country not surveyed + + Country not surveyed + + + + + NonInd + -999 + + + NonInd + 24.89999999999999858 + + + + + + #d9d9d9 + + + #000001 + 0.1 + bevel + + + + + + + +""" + +freshgwabs2_sld = """ + + + geonode:freshgwabs2 + + freshgwabs2 + 1 + + + < 1112 million cubic metres + < 1112 million cubic metres + + + + y2007 + 0 + + + y2007 + 1112 + + + + + + #ffe9b1 + + + #000001 + 0.1 + + + + + 1112 - 4794 million cubic metres + 1112 - 4794 million cubic metres + + + + y2007 + 1112 + + + y2007 + 4794 + + + + + + #eaad57 + + + #000001 + 0.1 + + + + + 4794 - 12096 million cubic metres + 4794 - 12096 million cubic metres + + + + y2007 + 4794 + + + y2007 + 12096 + + + + + + #ff7f1d + + + #000001 + 0.1 + + + + + 12096 - 28937 million cubic metres + 12096 - 28937 million cubic metres + + + + y2007 + 12096 + + + y2007 + 28937 + + + + + + #af8a33 + + + #000001 + 0.1 + + + + + > 28937 million cubic metres + > 28937 million cubic metres + + + + y2007 + 28937 + + + y2007 + 106910 + + + + + + #5b4b2d + + + #000001 + 0.1 + + + + + No data for 2007 + No data for 2007 + + + + y2007 + -99 + + + y2007 + 0 + + + + + + #d9d9d9 + + + #000001 + + + + + + + +""" + +raster_sld = """ + + + + + + + geonode-geonode_gwpollriskafriotest + + + + + + + grid + + 1 + + + + + + + + + + +""" + +line_sld = """ + + + line_3 + + line 3 + + + Single symbol + + + #db1e2a + 2 + round + round + 2 7 + + + + + + + +""" + +SLDS = { + "san_andres_y_providencia": san_andres_y_providencia_sld, + "lac": lac_sld, + "freshgwabs2": freshgwabs2_sld, + "raster": raster_sld, + "line": line_sld, +} + + +class LayerTests(GeoNodeBaseTestSupport): + type = "dataset" + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.user = "admin" + self.passwd = "admin" + create_dataset_data() + self.config = Configuration.load() + + self.OGC_DEFAULT_SETTINGS = { + "default": { + "BACKEND": "geonode.geoserver", + "LOCATION": "http://localhost:8080/geoserver/", + "USER": "admin", + "PASSWORD": "geoserver", + "MAPFISH_PRINT_ENABLED": True, + "PRINT_NG_ENABLED": True, + "GEONODE_SECURITY_ENABLED": True, + "GEOFENCE_SECURITY_ENABLED": True, + "GEOFENCE_TIMEOUT": 60, + "WMST_ENABLED": False, + "BACKEND_WRITE_ENABLED": True, + "WPS_ENABLED": False, + "DATASTORE": "", + "TIMEOUT": 60, + } + } + + self.UPLOADER_DEFAULT_SETTINGS = { + "BACKEND": "geonode.importer", + "OPTIONS": {"TIME_ENABLED": False, "MOSAIC_ENABLED": False}, + } + + self.DATABASE_DEFAULT_SETTINGS = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "development.db"}} + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_style_validity_and_name(self): + # Check that including an SLD with a valid shapefile results in the SLD + # getting picked up + d = None + try: + d = mkdtemp() + files = ( + "san_andres_y_providencia.sld", + "lac.sld", + "freshgwabs2.sld", + "raster.sld", + "line.sld", + ) + + for f in files: + path = os.path.join(d, f) + with open(path, "w") as f: + name, ext = splitext(basename(path)) + f.write(SLDS[name]) + + # Test 'san_andres_y_providencia.sld' + san_andres_y_providencia_sld_file = os.path.join(d, "san_andres_y_providencia.sld") + with open(san_andres_y_providencia_sld_file) as san_andres_y_providencia_sld_xml_file: + san_andres_y_providencia_sld_xml = san_andres_y_providencia_sld_xml_file.read() + san_andres_y_providencia_sld_name = extract_name_from_sld(None, san_andres_y_providencia_sld_xml) + self.assertEqual(san_andres_y_providencia_sld_name, "san_andres_y_providencia_administrative") + + # Test 'lac.sld' + lac_sld_file = os.path.join(d, "lac.sld") + with open(lac_sld_file) as lac_sld_xml_file: + lac_sld_xml = lac_sld_xml_file.read() + lac_sld_name = extract_name_from_sld(None, lac_sld_xml, sld_file=lac_sld_file) + self.assertEqual(lac_sld_name, "LAC NonIndigenous Access to Sanitation") + + # Test 'freshgwabs2.sld' + freshgwabs2_sld_file = os.path.join(d, "freshgwabs2.sld") + with open(freshgwabs2_sld_file) as freshgwabs2_sld_xml_file: + freshgwabs2_sld_xml = freshgwabs2_sld_xml_file.read() + freshgwabs2_sld_name = extract_name_from_sld(None, freshgwabs2_sld_xml, sld_file=freshgwabs2_sld_file) + self.assertEqual(freshgwabs2_sld_name, "freshgwabs2") + + # Test 'raster.sld' + raster_sld_file = os.path.join(d, "raster.sld") + with open(raster_sld_file) as raster_sld_xml_file: + raster_sld_xml = raster_sld_xml_file.read() + raster_sld_name = extract_name_from_sld(None, raster_sld_xml, sld_file=raster_sld_file) + self.assertEqual(raster_sld_name, "geonode-geonode_gwpollriskafriotest") + + # Test 'line.sld' + line_sld_file = os.path.join(d, "line.sld") + with open(line_sld_file) as line_sld_xml_file: + line_sld_xml = line_sld_xml_file.read() + line_sld_name = extract_name_from_sld(None, line_sld_xml, sld_file=line_sld_file) + self.assertEqual(line_sld_name, "line 3") + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_style_change_on_basic_auth(self): + """ + Ensures we are able to update the style through a BASIC auth call only. + """ + layer = Dataset.objects.filter(default_style__isnull=False).first() + + bob = get_user_model().objects.get(username="bobby") + assign_perm("change_dataset_style", bob, layer) + + self.assertTrue(bob.has_perm("change_dataset_style", obj=layer)) + + # Test that HTTP_AUTHORIZATION in request.META is working properly + valid_uname_pw = b"bobby:bob" + invalid_uname_pw = b"n0t:v@l1d" + + valid_auth_headers = { + "HTTP_AUTHORIZATION": f"BASIC {base64.b64encode(valid_uname_pw).decode()}", + } + + invalid_auth_headers = { + "HTTP_AUTHORIZATION": f"BASIC {base64.b64encode(invalid_uname_pw).decode()}", + } + + change_style_url = urljoin( + settings.SITEURL, f"/gs/rest/workspaces/{settings.DEFAULT_WORKSPACE}/styles/{layer.name}?raw=true" + ) + logger.debug(f"{change_style_url}") + + rf = RequestFactory() + + # Check is 'authorized' + post_request = rf.post( + change_style_url, + data=san_andres_y_providencia_sld, + content_type="application/vnd.ogc.sld+xml", + **valid_auth_headers, + ) + post_request.user = AnonymousUser() + raw_url, headers, access_token, downstream_path = check_geoserver_access( + post_request, + "/gs/rest/workspaces", + "rest/workspaces", + workspace="geonode", + layername=layer.name, + allowed_hosts=[ + urlsplit(ogc_server_settings.public_url).hostname, + ], + ) + self.assertIsNotNone(raw_url) + self.assertIsNotNone(headers) + self.assertIsNotNone(access_token) + + authorized = style_change_check(post_request, downstream_path, style_name="styles", access_token=access_token) + self.assertTrue(authorized) + + authorized = style_change_check( + post_request, "rest/styles", style_name=f"{layer.name}", access_token=access_token + ) + self.assertTrue(authorized) + + authorized = style_change_check( + post_request, + f"rest/workspaces/{layer.workspace}/styles/{layer.name}", + style_name=f"{layer.name}", + access_token=access_token, + ) + self.assertTrue(authorized) + + authorized = style_change_check(post_request, f"rest/layers/{layer.name}", access_token=access_token) + self.assertTrue(authorized) + + authorized = style_change_check( + post_request, f"rest/workspaces/{layer.workspace}/layers/{layer.name}", access_token=access_token + ) + self.assertTrue(authorized) + + put_request = rf.put( + change_style_url, + data=san_andres_y_providencia_sld, + content_type="application/vnd.ogc.sld+xml", + **valid_auth_headers, + ) + put_request.user = AnonymousUser() + raw_url, headers, access_token, downstream_path = check_geoserver_access( + put_request, + "/gs/rest/workspaces", + "rest/workspaces", + workspace="geonode", + layername=layer.name, + allowed_hosts=[ + urlsplit(ogc_server_settings.public_url).hostname, + ], + ) + self.assertIsNotNone(raw_url) + self.assertIsNotNone(headers) + self.assertIsNotNone(access_token) + + # Check that, if we have been authorized through the "access_token", + # we can still update a style no more present on GeoNode + # ref: 05b000cdb06b0b6e9b72bd9eb8a8e03abeb204a8 + # [Regression] "style_change_check" always fails in the case the style does not exist on GeoNode too, preventing a user editing temporary generated styles + Style.objects.filter(name=layer.name).delete() + + authorized = style_change_check(put_request, downstream_path, style_name="styles", access_token=access_token) + self.assertTrue(authorized) + + authorized = style_change_check( + put_request, "rest/styles", style_name=f"{layer.name}", access_token=access_token + ) + self.assertTrue(authorized) + + authorized = style_change_check( + put_request, + f"rest/workspaces/{layer.workspace}/styles/{layer.name}", + style_name=f"{layer.name}", + access_token=access_token, + ) + self.assertTrue(authorized) + + authorized = style_change_check(put_request, f"rest/layers/{layer.name}", access_token=access_token) + self.assertTrue(authorized) + + authorized = style_change_check( + put_request, f"rest/workspaces/{layer.workspace}/layers/{layer.name}", access_token=access_token + ) + self.assertTrue(authorized) + + # Check is NOT 'authorized' + post_request = rf.post( + change_style_url, + data=san_andres_y_providencia_sld, + content_type="application/vnd.ogc.sld+xml", + **invalid_auth_headers, + ) + post_request.user = AnonymousUser() + raw_url, headers, access_token, downstream_path = check_geoserver_access( + post_request, + "/gs/rest/workspaces", + "rest/workspaces", + workspace="geonode", + layername=layer.name, + allowed_hosts=[ + urlsplit(ogc_server_settings.public_url).hostname, + ], + ) + self.assertIsNotNone(raw_url) + self.assertIsNotNone(headers) + self.assertIsNone(access_token) + + authorized = style_change_check(post_request, downstream_path, access_token=access_token) + self.assertFalse(authorized) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_dataset_acls(self): + """Verify that the dataset_acls view is behaving as expected""" + + # Test that HTTP_AUTHORIZATION in request.META is working properly + valid_uname_pw = b"bobby:bob" + invalid_uname_pw = b"n0t:v@l1d" + + valid_auth_headers = { + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(valid_uname_pw).decode()}", + } + + invalid_auth_headers = { + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(invalid_uname_pw).decode()}", + } + + bob = get_user_model().objects.get(username="bobby") + dataset_ca = Dataset.objects.get(alternate="geonode:CA") + assign_perm("change_dataset_data", bob, dataset_ca) + + # Test that requesting when supplying the geoserver credentials returns + # the expected json + + expected_result = { + "email": "bobby@bob.com", + "fullname": "bobby", + "is_anonymous": False, + "is_superuser": False, + "name": "bobby", + "ro": [ + "geonode:layer2", + "geonode:mylayer", + "geonode:foo", + "geonode:whatever", + "geonode:fooey", + "geonode:quux", + "geonode:fleem", + ], + "rw": ["geonode:CA"], + } + response = self.client.get(reverse("dataset_acls"), **valid_auth_headers) + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + response_json = json.loads(content) + # 'ro' and 'rw' are unsorted collections + self.assertEqual(sorted(expected_result), sorted(response_json)) + + # Test that requesting when supplying invalid credentials returns the + # appropriate error code + response = self.client.get(reverse("dataset_acls"), **invalid_auth_headers) + self.assertEqual(response.status_code, 401) + + # Test logging in using Djangos normal auth system + self.client.login(username="admin", password="admin") + + # Basic check that the returned content is at least valid json + response = self.client.get(reverse("dataset_acls")) + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + response_json = json.loads(content) + + self.assertEqual("admin", response_json["fullname"]) + self.assertEqual("ad@m.in", response_json["email"]) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_resolve_user(self): + """Verify that the resolve_user view is behaving as expected""" + # Test that HTTP_AUTHORIZATION in request.META is working properly + valid_uname_pw = b"admin:admin" + invalid_uname_pw = b"n0t:v@l1d" + + valid_auth_headers = { + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(valid_uname_pw).decode()}", + } + + invalid_auth_headers = { + "HTTP_AUTHORIZATION": f"basic {base64.b64encode(invalid_uname_pw).decode()}", + } + + response = self.client.get(reverse("dataset_resolve_user_dep"), **valid_auth_headers) + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + response_json = json.loads(content) + self.assertEqual( + {"geoserver": False, "superuser": True, "user": "admin", "fullname": "admin", "email": "ad@m.in"}, + response_json, + ) + + # Test that requesting when supplying invalid credentials returns the + # appropriate error code + response = self.client.get(reverse("dataset_acls"), **invalid_auth_headers) + self.assertEqual(response.status_code, 401) + + # Test logging in using Djangos normal auth system + self.client.login(username="admin", password="admin") + + # Basic check that the returned content is at least valid json + response = self.client.get(reverse("dataset_resolve_user_dep")) + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + response_json = json.loads(content) + + self.assertEqual("admin", response_json["user"]) + self.assertEqual("admin", response_json["fullname"]) + self.assertEqual("ad@m.in", response_json["email"]) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_ogc_server_settings(self): + """ + Tests the OGC Servers Handler class. + """ + + with override_settings(OGC_SERVER=self.OGC_DEFAULT_SETTINGS, UPLOADER=self.UPLOADER_DEFAULT_SETTINGS): + OGC_SERVER = self.OGC_DEFAULT_SETTINGS.copy() + OGC_SERVER.update({"PUBLIC_LOCATION": "http://geoserver:8080/geoserver/"}) + + ogc_settings = OGC_Servers_Handler(OGC_SERVER)["default"] + + default = OGC_SERVER.get("default") + self.assertEqual(ogc_settings.server, default) + self.assertEqual(ogc_settings.BACKEND, default.get("BACKEND")) + self.assertEqual(ogc_settings.LOCATION, default.get("LOCATION")) + self.assertEqual(ogc_settings.PUBLIC_LOCATION, default.get("PUBLIC_LOCATION")) + self.assertEqual(ogc_settings.USER, default.get("USER")) + self.assertEqual(ogc_settings.PASSWORD, default.get("PASSWORD")) + self.assertEqual(ogc_settings.DATASTORE, "") + self.assertEqual(ogc_settings.credentials, ("admin", "geoserver")) + self.assertTrue(ogc_settings.MAPFISH_PRINT_ENABLED) + self.assertTrue(ogc_settings.PRINT_NG_ENABLED) + self.assertTrue(ogc_settings.GEONODE_SECURITY_ENABLED) + self.assertFalse(ogc_settings.WMST_ENABLED) + self.assertTrue(ogc_settings.BACKEND_WRITE_ENABLED) + self.assertFalse(ogc_settings.WPS_ENABLED) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_ogc_server_defaults(self): + """ + Tests that OGC_SERVER_SETTINGS are built if they do not exist in the settings. + """ + from django.urls import reverse, resolve + from ..ows import _wcs_get_capabilities, _wfs_get_capabilities, _wms_get_capabilities + + OGC_SERVER = {"default": dict()} + + defaults = self.OGC_DEFAULT_SETTINGS.get("default") + ogc_settings = OGC_Servers_Handler(OGC_SERVER)["default"] + self.assertEqual(ogc_settings.server, defaults) + self.assertEqual(ogc_settings.rest, f"{defaults['LOCATION']}rest") + self.assertEqual(ogc_settings.ows, f"{defaults['LOCATION']}ows") + + # Make sure we get None vs a KeyError when the key does not exist + self.assertIsNone(ogc_settings.SFDSDFDSF) + + # Testing REST endpoints + route = resolve("/gs/rest/layers").route + self.assertEqual(route, "^gs/rest/layers") + + route = resolve("/gs/rest/imports").route + self.assertEqual(route, "^gs/rest/imports") + + route = resolve("/gs/rest/sldservice").route + self.assertEqual(route, "^gs/rest/sldservice") + + store_resolver = resolve("/gs/rest/stores/geonode_data/") + self.assertEqual(store_resolver.url_name, "gs_stores") + self.assertEqual(store_resolver.kwargs["store_type"], "geonode_data") + self.assertEqual(store_resolver.route, "^gs/rest/stores/(?P\\w+)/$") + + sld_resolver = resolve("/gs/rest/styles") + self.assertEqual(sld_resolver.url_name, "gs_styles") + self.assertTrue("workspace" not in sld_resolver.kwargs) + self.assertEqual(sld_resolver.kwargs["proxy_path"], "/gs/rest/styles") + self.assertEqual(sld_resolver.kwargs["downstream_path"], "rest/styles") + self.assertEqual(sld_resolver.route, "^gs/rest/styles") + + sld_resolver = resolve("/gs/rest/workspaces/geonode/styles") + self.assertEqual(sld_resolver.url_name, "gs_workspaces") + self.assertEqual(sld_resolver.kwargs["workspace"], "geonode") + self.assertEqual(sld_resolver.kwargs["proxy_path"], "/gs/rest/workspaces") + self.assertEqual(sld_resolver.kwargs["downstream_path"], "rest/workspaces") + self.assertEqual(sld_resolver.route, "^gs/rest/workspaces/(?P\\w+)") + + # Testing OWS endpoints + wcs = _wcs_get_capabilities() + logger.debug(wcs) + self.assertIsNotNone(wcs) + + try: + wcs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) + except Exception: + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + + self.assertTrue(wcs.startswith(wcs_url)) + self.assertIn("service=WCS", wcs) + self.assertIn("request=GetCapabilities", wcs) + self.assertIn("version=2.0.1", wcs) + + wfs = _wfs_get_capabilities() + logger.debug(wfs) + self.assertIsNotNone(wfs) + + try: + wfs_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) + except Exception: + wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + self.assertTrue(wfs.startswith(wfs_url)) + self.assertIn("service=WFS", wfs) + self.assertIn("request=GetCapabilities", wfs) + self.assertIn("version=1.1.0", wfs) + + wms = _wms_get_capabilities() + logger.debug(wms) + self.assertIsNotNone(wms) + + try: + wms_url = urljoin(settings.SITEURL, reverse("ows_endpoint")) + except Exception: + wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, "ows") + self.assertTrue(wms.startswith(wms_url)) + self.assertIn("service=WMS", wms) + self.assertIn("request=GetCapabilities", wms) + self.assertIn("version=1.3.0", wms) + + # Test OWS Download Links + from geonode.geoserver.ows import wcs_links, wfs_links, wms_links + + instance = create_single_dataset("san_andres_y_providencia_water") + instance.name = "san_andres_y_providencia_water" + instance.save() + bbox = instance.bbox + srid = instance.srid + height = 512 + width = 512 + + # Default Style (expect exception since we are offline) + style = get_sld_for(gs_catalog, instance) + if style and isinstance(style, str): + style = gs_catalog.get_style(instance.name, workspace=instance.workspace) + self.assertIsNotNone(style) + self.assertFalse(isinstance(style, str)) + style_name = "point" + style_body = """ + + + + + default_point + + + Default Point + A sample style that draws a point + + + + + rule1 + Red Square + A 6 pixel square with a red fill and no stroke + + + + square + + #FF0000 + + + 6 + + + + + + + +""" + instance.default_style, _ = Style.objects.get_or_create( + name=style_name, defaults=dict(sld_title=style_name, sld_body=style_body) + ) + self.assertIsNotNone(instance.default_style) + self.assertIsNotNone(instance.default_style.name) + + # WMS Links + wms_links = wms_links(f"{ogc_settings.public_url}wms?", instance.alternate, bbox, srid, height, width) + self.assertIsNotNone(wms_links) + self.assertEqual(len(wms_links), 3) + wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, "wms") + identifier = urlencode({"layers": instance.alternate}) + for _link in wms_links: + logger.debug(f"{wms_url} --> {_link[3]}") + self.assertTrue(wms_url in _link[3]) + logger.debug(f"{identifier} --> {_link[3]}") + self.assertTrue(identifier in _link[3]) + + # WFS Links + wfs_links = wfs_links(f"{ogc_settings.public_url}wfs?", instance.alternate, bbox, srid) + self.assertIsNotNone(wfs_links) + self.assertEqual(len(wfs_links), 6) + wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "wfs") + identifier = urlencode({"typename": instance.alternate}) + for _link in wfs_links: + logger.debug(f"{wfs_url} --> {_link[3]}") + self.assertTrue(wfs_url in _link[3]) + logger.debug(f"{identifier} --> {_link[3]}") + self.assertTrue(identifier in _link[3]) + + # WCS Links + wcs_links = wcs_links(f"{ogc_settings.public_url}wcs?", instance.alternate) + self.assertIsNotNone(wcs_links) + self.assertEqual(len(wcs_links), 1) + wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, "wcs") + identifier = urlencode({"coverageid": instance.alternate.replace(":", "__", 1)}) + for _link in wcs_links: + logger.debug(f"{wcs_url} --> {_link[3]}") + self.assertTrue(wcs_url in _link[3]) + logger.debug(f"{identifier} --> {_link[3]}") + self.assertTrue(identifier in _link[3]) + if srid: + self.assertFalse("outputCrs" in _link[3]) + if bbox: + self.assertFalse("subset=Long" in _link[3]) + self.assertFalse("subset=Lat" in _link[3]) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_importer_configuration(self): + database_settings = self.DATABASE_DEFAULT_SETTINGS.copy() + ogc_server_settings = self.OGC_DEFAULT_SETTINGS.copy() + uploader_settings = self.UPLOADER_DEFAULT_SETTINGS.copy() + + uploader_settings["BACKEND"] = "geonode.importer" + self.assertTrue(["geonode_imports" not in database_settings.keys()]) + + # Test the importer backend without specifying a datastore or + # corresponding database. + with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): + OGC_Servers_Handler(ogc_server_settings)["default"] + + ogc_server_settings["default"]["DATASTORE"] = "geonode_imports" + + # Test the importer backend with a datastore but no corresponding + # database. + with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): + OGC_Servers_Handler(ogc_server_settings)["default"] + + database_settings["geonode_imports"] = database_settings["default"].copy() + database_settings["geonode_imports"].update({"NAME": "geonode_imports"}) + + # Test the importer backend with a datastore and a corresponding + # database, no exceptions should be thrown. + with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings): + OGC_Servers_Handler(ogc_server_settings)["default"] + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_set_resources_links(self): + from django.db.models import Q + from geonode.base.models import Link + from geonode.catalogue import get_catalogue + + with self.settings(UPDATE_RESOURCE_LINKS_AT_MIGRATE=True, ASYNC_SIGNALS=False): + # Links + _def_link_types = ["original", "metadata"] + # Check 'original' and 'metadata' links exist + Link.objects.update_or_create( + resource=Dataset.objects.first(), + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) + _links = Link.objects.filter(link_type__in=_def_link_types) + + self.assertIsNotNone(_links, "No 'original' and 'metadata' links have been found") + + # Delete all 'original' and 'metadata' links + _links.delete() + self.assertFalse(_links.exists(), "No links have been deleted") + # Delete resources metadata + _datasets = Dataset.objects.exclude( + Q(metadata_xml__isnull=True) + | Q(metadata_xml__exact="") + | Q(csw_anytext__isnull=True) + | Q(csw_anytext__exact="") + ) + count = _datasets.count() + if count: + _datasets.update(metadata_xml=None) + _updated_datasets = Dataset.objects.exclude( + Q(metadata_xml__isnull=True) + | Q(metadata_xml__exact="") + | Q(csw_anytext__isnull=True) + | Q(csw_anytext__exact="") + ) + updated_count = _updated_datasets.count() + self.assertTrue(updated_count == 0, "Metadata have not been updated (deleted) correctly") + + # Call migrate + call_command("migrate", verbosity=0) + # Check links + _post_migrate_links = Link.objects.filter(link_type__in=_def_link_types) + self.assertTrue(_post_migrate_links.exists(), "No links have been restored") + # Check layers + _post_migrate_datasets = Dataset.objects.exclude( + Q(metadata_xml__isnull=True) + | Q(metadata_xml__exact="") + | Q(csw_anytext__isnull=True) + | Q(csw_anytext__exact="") + ) + + for _lyr in _post_migrate_datasets: + # Check original links in csw_anytext + # by default is not created anymore, we need to create one + Link.objects.update_or_create( + resource=_lyr, + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) + + _post_migrate_links_orig = Link.objects.filter( + resource=_lyr.resourcebase_ptr, resource_id=_lyr.resourcebase_ptr.id, link_type="original" + ) + self.assertTrue( + _post_migrate_links_orig.exists(), + f"No 'original' links has been found for the layer '{_lyr.alternate}'", + ) + # needed to update the csw_anytext field with the new link created + catalogue_post_save(instance=_lyr, sender=_lyr.__class__) + _lyr.refresh_from_db() + for _link_orig in _post_migrate_links_orig: + self.assertIn( + _link_orig.url, + _lyr.csw_anytext, + f"The link URL {_link_orig.url} is not present in the 'csw_anytext' attribute of the layer '{_lyr.alternate}'", + ) + # Check catalogue + catalogue = get_catalogue() + record = catalogue.get_record(_lyr.uuid) + self.assertIsNotNone(record) + self.assertTrue( + hasattr(record, "links"), + f"No records have been found in the catalogue for the resource '{_lyr.alternate}'", + ) + # Check 'metadata' links for each record + for mime, name, metadata_url in record.links["metadata"]: + try: + _post_migrate_link_meta = Link.objects.get( + resource=_lyr.resourcebase_ptr, + url=metadata_url, + name=name, + extension="xml", + mime=mime, + link_type="metadata", + ) + except Link.DoesNotExist: + _post_migrate_link_meta = None + self.assertIsNotNone( + _post_migrate_link_meta, + f"No '{name}' links have been found in the catalogue for the resource '{_lyr.alternate}'", + ) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_gs_proxy_never_caches(self): + url = reverse("gs_styles") + response = self.client.get(url) + self.assertFalse(response.has_header("Cache-Control")) diff --git a/geonode/geoserver/tests/test_tasks.py b/geonode/geoserver/tests/test_tasks.py new file mode 100644 index 0000000..8945390 --- /dev/null +++ b/geonode/geoserver/tests/test_tasks.py @@ -0,0 +1,82 @@ +from unittest.mock import create_autospec, patch + +from geonode.base.populate_test_data import all_public, create_models, remove_models +from geonode.geoserver.tasks import geoserver_create_style, geoserver_set_style +from geonode.geoserver.signals import geoserver_automatic_default_style_set +from geonode.layers.models import Dataset +from geonode.layers.populate_datasets_data import create_dataset_data +from geonode.tests.base import GeoNodeBaseTestSupport + + +class TasksTest(GeoNodeBaseTestSupport): + type = "dataset" + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + create_dataset_data() + + def mock_signal_callback(self, instance, **kwargs): + pass + + def test_geoserver_style_visual_mode_automatically_with_sld_file(self): + dataset = Dataset.objects.first() + sld_file = "geonode/base/fixtures/test_sld.sld" + handler = create_autospec(self.mock_signal_callback) + + geoserver_automatic_default_style_set.connect(handler) + with patch("geoserver.catalog.Catalog.get_style") as style_mck: + style_mck.return_value = True + geoserver_create_style(dataset.id, dataset.name, sld_file=sld_file, tempdir=None) + self.assertEqual(handler.call_count, 0) + + def test_geoserver_style_visual_mode_automatically_without_sld_file(self): + dataset = Dataset.objects.first() + handler = create_autospec(self.mock_signal_callback) + + geoserver_automatic_default_style_set.connect(handler) + geoserver_create_style(dataset.id, dataset.name, sld_file=None, tempdir=None) + handler.assert_called_once_with(signal=geoserver_automatic_default_style_set, sender=dataset, instance=dataset) + + @patch("geonode.geoserver.tasks.set_dataset_style") + def test_geoserver_set_style_with_real_file(self, mocked_set_dataset_style): + dataset = Dataset.objects.first() + sld_file = "geonode/base/fixtures/test_sld.sld" + geoserver_set_style(instance_id=dataset.id, base_file=sld_file) + mocked_set_dataset_style.assert_called_once() + + args_list = mocked_set_dataset_style.call_args_list[0].args + kwargs_list = mocked_set_dataset_style.call_args_list[0].kwargs + self.assertEqual(args_list[0].id, dataset.id) + self.assertEqual(args_list[1], dataset.alternate) + self.assertIsInstance(args_list[2], bytes) + + self.assertDictEqual({"base_file": sld_file}, kwargs_list) + + @patch("geonode.geoserver.tasks.set_dataset_style") + def test_geoserver_set_style_with_xml(self, mocked_set_dataset_style): + dataset = Dataset.objects.first() + + with open("geonode/base/fixtures/test_sld.sld", "r+") as _file: + geoserver_set_style(instance_id=dataset.id, base_file=_file.read()) + mocked_set_dataset_style.assert_called_once() + + args_list = mocked_set_dataset_style.call_args_list[0].args + kwargs_list = mocked_set_dataset_style.call_args_list[0].kwargs + self.assertEqual(args_list[0].id, dataset.id) + self.assertEqual(args_list[1], dataset.alternate) + self.assertIsInstance(args_list[2], str) + + self.assertDictEqual({"base_file": None}, kwargs_list) diff --git a/geonode/geoserver/upload.py b/geonode/geoserver/upload.py new file mode 100644 index 0000000..d5c9dde --- /dev/null +++ b/geonode/geoserver/upload.py @@ -0,0 +1,263 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import uuid +import logging +import geoserver + +from geoserver.catalog import ConflictingDataError, UploadError +from geoserver.resource import FeatureType, Coverage + +from django.conf import settings + +from geonode import GeoNodeException +from geonode.layers.utils import dataset_type, get_files +from .helpers import ( + GEOSERVER_LAYER_TYPES, + gs_catalog, + get_store, + get_sld_for, + ogc_server_settings, + _create_db_featurestore, + _create_featurestore, + _create_coveragestore, +) + +logger = logging.getLogger(__name__) + + +def geoserver_dataset_type(filename): + the_type = dataset_type(filename) + return GEOSERVER_LAYER_TYPES[the_type] + + +def geoserver_upload( + dataset, + base_file, + user, + name, + overwrite=True, + title=None, + abstract=None, + permissions=None, + keywords=(), + charset="UTF-8", +): + # Step 2. Check that it is uploading to the same resource type as + # the existing resource + logger.debug( + ">>> Step 2. Make sure we are not trying to overwrite a " "existing resource named [%s] with the wrong type", + name, + ) + the_dataset_type = geoserver_dataset_type(base_file) + + # Get a short handle to the gsconfig geoserver catalog + cat = gs_catalog + + # Ahmed Nour: get workspace by name instead of get default one. + workspace = cat.get_workspace(settings.DEFAULT_WORKSPACE) + # Check if the store exists in geoserver + try: + store = get_store(cat, name, workspace=workspace) + except geoserver.catalog.FailedRequestError: + # There is no store, ergo the road is clear + pass + else: + # If we get a store, we do the following: + resources = cat.get_resources(names=[name], stores=[store], workspaces=[workspace]) + + if len(resources) > 0: + # If our resource is already configured in the store it needs + # to have the right resource type + for resource in resources: + if resource.name == name: + msg = "Name already in use and overwrite is False" + assert overwrite, msg + existing_type = resource.resource_type + if existing_type != the_dataset_type: + msg = ( + f"Type of uploaded file {name} ({the_dataset_type}) " + "does not match type of existing " + f"resource type {existing_type}" + ) + logger.debug(msg) + raise GeoNodeException(msg) + + # Step 3. Identify whether it is vector or raster and which extra files + # are needed. + logger.debug(">>> Step 3. Identifying if [%s] is vector or raster and " "gathering extra files", name) + if the_dataset_type == FeatureType.resource_type: + logger.debug("Uploading vector layer: [%s]", base_file) + if ogc_server_settings.DATASTORE: + create_store_and_resource = _create_db_featurestore + else: + create_store_and_resource = _create_featurestore + elif the_dataset_type == Coverage.resource_type: + logger.debug("Uploading raster layer: [%s]", base_file) + create_store_and_resource = _create_coveragestore + else: + msg = ( + f"The layer type for name {name} is {the_dataset_type}. It should be " + f"{FeatureType.resource_type} or {Coverage.resource_type}," + ) + logger.warn(msg) + raise GeoNodeException(msg) + + # Step 4. Create the store in GeoServer + logger.debug(">>> Step 4. Starting upload of [%s] to GeoServer...", name) + + # Get the helper files if they exist + files, _tmpdir = get_files(base_file) + data = files + if "shp" not in files: + data = base_file + try: + store, gs_resource = create_store_and_resource( + name, data, charset=charset, overwrite=overwrite, workspace=workspace + ) + except UploadError as e: + msg = f"Could not save the layer {name}, there was an upload " f"error: {e}" + logger.warn(msg) + e.args = (msg,) + raise + except ConflictingDataError as e: + # A datastore of this name already exists + msg = ( + f"GeoServer reported a conflict creating a store with name {name}: " + f'"{e}". This should never happen because a brand new name ' + "should have been generated. But since it happened, " + "try renaming the file or deleting the store in GeoServer." + ) + logger.warn(msg) + e.args = (msg,) + raise + except Exception as e: + logger.error("Error during the creation of the resource in GeoServer", exc_info=e) + raise e + + logger.debug(f"The File {name} has been sent to GeoServer without errors.") + + # Step 5. Create the resource in GeoServer + logger.debug(f">>> Step 5. Generating the metadata for {name} after successful import to GeoSever") + + # Verify the resource was created + if not gs_resource: + gs_resource = gs_catalog.get_resource(name=name, workspace=workspace) + + if not gs_resource: + msg = f"GeoNode encountered problems when creating layer {name}.It cannot find the Dataset that matches this Workspace.try renaming your files." + logger.warn(msg) + raise GeoNodeException(msg) + + assert gs_resource.name == name + + # Step 6. Make sure our data always has a valid projection + logger.debug(f">>> Step 6. Making sure [{name}] has a valid projection") + _native_bbox = None + try: + _native_bbox = gs_resource.native_bbox + except Exception: + pass + + if _native_bbox and len(_native_bbox) >= 5 and _native_bbox[4:5][0] == "EPSG:4326": + box = _native_bbox[:4] + minx, maxx, miny, maxy = [float(a) for a in box] + if ( + -180 <= round(minx, 5) <= 180 + and -180 <= round(maxx, 5) <= 180 + and -90 <= round(miny, 5) <= 90 + and -90 <= round(maxy, 5) <= 90 + ): + gs_resource.latlon_bbox = _native_bbox + gs_resource.projection = "EPSG:4326" + else: + logger.warning("BBOX coordinates outside normal EPSG:4326 values for layer " "[%s].", name) + _native_bbox = [-180, -90, 180, 90, "EPSG:4326"] + gs_resource.latlon_bbox = _native_bbox + gs_resource.projection = "EPSG:4326" + logger.debug("BBOX coordinates forced to [-180, -90, 180, 90] for layer [%s].", name) + + # Step 7. Create the style and assign it to the created resource + logger.debug(f">>> Step 7. Creating style for [{name}]") + cat.save(gs_resource) + publishing = cat.get_layer(name) or gs_resource + sld = None + try: + if "sld" in files: + with open(files["sld"], "rb") as f: + sld = f.read() + else: + sld = get_sld_for(cat, dataset) + except Exception as e: + logger.exception(e) + + style = None + if sld: + try: + style = cat.get_style(name, workspace=workspace) + except geoserver.catalog.FailedRequestError: + style = cat.get_style(name) + + try: + overwrite = style or False + cat.create_style(name, sld, overwrite=overwrite, raw=True, workspace=workspace) + cat.reset() + except geoserver.catalog.ConflictingDataError as e: + msg = f"There was already a style named {name}_dataset in GeoServer, " f'try to use: "{e}"' + logger.warn(msg) + e.args = (msg,) + except geoserver.catalog.UploadError as e: + msg = f"Error while trying to upload style named {name}_dataset in GeoServer, " f'try to use: "{e}"' + e.args = (msg,) + logger.exception(e) + + if style is None: + try: + style = cat.get_style(name, workspace=workspace) or cat.get_style(name) + except Exception as e: + style = cat.get_style("point") + msg = f'Could not find any suitable style in GeoServer for Dataset: "{name}"' + e.args = (msg,) + logger.exception(e) + + if style: + publishing.default_style = style + logger.debug("default style set to %s", name) + try: + cat.save(publishing) + except geoserver.catalog.FailedRequestError as e: + msg = f"Error while trying to save resource named {publishing} in GeoServer, " f'try to use: "{e}"' + e.args = (msg,) + logger.exception(e) + + # Step 8. Create the Django record for the layer + logger.debug(">>> Step 8. Creating Django record for [%s]", name) + alternate = f"{workspace.name}:{gs_resource.name}" + dataset_uuid = str(uuid.uuid4()) + + defaults = dict( + store=gs_resource.store.name, + subtype=gs_resource.store.resource_type, + alternate=alternate, + title=title or gs_resource.title, + uuid=dataset_uuid, + abstract=abstract or gs_resource.abstract or "", + owner=user, + ) + + return name, workspace.name, defaults, gs_resource diff --git a/geonode/geoserver/urls.py b/geonode/geoserver/urls.py new file mode 100644 index 0000000..b35f176 --- /dev/null +++ b/geonode/geoserver/urls.py @@ -0,0 +1,88 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.conf import settings +from django.urls import re_path +from . import views + +urlpatterns = [ # 'geonode.geoserver.views', + # REST Endpoints + re_path(r"^rest/stores/(?P\w+)/$", views.stores, name="gs_stores"), + re_path( + r"^rest/styles", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/styles", downstream_path="rest/styles"), + name="gs_styles", + ), + re_path( + r"^rest/workspaces/(?P\w+)", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/workspaces", downstream_path="rest/workspaces"), + name="gs_workspaces", + ), + re_path( + r"^rest/layers", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/layers", downstream_path="rest/layers"), + name="gs_layers", + ), + re_path( + r"^rest/imports", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/imports", downstream_path="rest/imports"), + name="gs_imports", + ), + re_path( + r"^rest/sldservice", + views.geoserver_proxy, + dict(proxy_path="/gs/rest/sldservice", downstream_path="rest/sldservice"), + name="gs_sldservice", + ), + # OWS Endpoints + re_path(r"^ows", views.geoserver_proxy, dict(proxy_path="/gs/ows", downstream_path="ows"), name="ows_endpoint"), + re_path(r"^gwc", views.geoserver_proxy, dict(proxy_path="/gs/gwc", downstream_path="gwc"), name="gwc_endpoint"), + re_path(r"^wms", views.geoserver_proxy, dict(proxy_path="/gs/wms", downstream_path="wms"), name="wms_endpoint"), + re_path(r"^wfs", views.geoserver_proxy, dict(proxy_path="/gs/wfs", downstream_path="wfs"), name="wfs_endpoint"), + re_path(r"^wcs", views.geoserver_proxy, dict(proxy_path="/gs/wcs", downstream_path="wcs"), name="wcs_endpoint"), + re_path(r"^wps", views.geoserver_proxy, dict(proxy_path="/gs/wps", downstream_path="wps"), name="wps_endpoint"), + re_path(r"^pdf", views.geoserver_proxy, dict(proxy_path="/gs/pdf", downstream_path="pdf"), name="pdf_endpoint"), + re_path( + r"^(?P[^/]*)/(?P[^/]*)/ows", + views.geoserver_proxy, + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="ows"), + ), + re_path( + r"^(?P[^/]*)/(?P[^/]*)/wms", + views.geoserver_proxy, + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="wms"), + ), + re_path( + r"^(?P[^/]*)/(?P[^/]*)/wfs", + views.geoserver_proxy, + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="wfs"), + ), + re_path( + r"^(?P[^/]*)/(?P[^/]*)/wcs", + views.geoserver_proxy, + dict(proxy_path=f"/gs/{settings.DEFAULT_WORKSPACE}", downstream_path="wcs"), + ), + re_path(r"^updatelayers/$", views.updatelayers, name="updatelayers"), + re_path(r"^acls/?$", views.dataset_acls, name="dataset_acls"), + re_path(r"^online/?$", views.server_online, name="server_online"), +] diff --git a/geonode/geoserver/views.py b/geonode/geoserver/views.py new file mode 100644 index 0000000..0a601a9 --- /dev/null +++ b/geonode/geoserver/views.py @@ -0,0 +1,573 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import re +import json +import logging +import warnings +from lxml import etree +from owslib.etree import etree as dlxml + +from urllib.parse import urlsplit, urljoin, unquote, parse_qsl + +from django.contrib.auth import authenticate +from django.http import HttpResponse, HttpResponseRedirect +from django.views.decorators.http import require_POST +from django.conf import settings +from django.contrib.auth.decorators import user_passes_test +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required +from django.template.loader import get_template +from django.utils.translation import gettext_lazy as _ + +from guardian.shortcuts import get_objects_for_user + +from geonode.base.models import ResourceBase +from geonode.client.hooks import hookset +from geonode.compat import ensure_string +from geonode.base.auth import get_auth_user, get_or_create_token +from geonode.decorators import logged_in_or_basicauth +from geonode.layers.models import Dataset, Style +from geonode.layers.views import _resolve_dataset, _PERMISSION_MSG_MODIFY +from geonode.maps.models import Map +from geonode.proxy.views import proxy, fetch_response_headers +from .tasks import geoserver_update_datasets +from geonode.utils import ( + _get_basic_auth_info, + http_client, + get_headers, +) +from geonode.geoserver.signals import geoserver_post_save_local +from .helpers import ( + get_stores, + ogc_server_settings, + style_update, + ows_endpoint_in_path, + temp_style_name_regex, + get_dataset_capabilities_url, + _stylefilterparams_geowebcache_dataset, + _invalidate_geowebcache_dataset, +) + +from django.views.decorators.csrf import csrf_exempt + +logger = logging.getLogger(__name__) + + +def stores(request, store_type=None): + stores = get_stores(store_type) + data = json.dumps(stores) + return HttpResponse(data) + + +@user_passes_test(lambda u: u.is_superuser) +def updatelayers(request): + params = request.GET + # Get the owner specified in the request if any, otherwise used the logged + # user + owner = params.get("owner", None) + owner = get_user_model().objects.get(username=owner) if owner is not None else request.user + workspace = params.get("workspace", None) + store = params.get("store", None) + filter = params.get("filter", None) + result = geoserver_update_datasets.delay( + ignore_errors=False, owner=owner, workspace=workspace, store=store, filter=filter + ) + # Attempt to run task synchronously + result.get() + + return HttpResponseRedirect(hookset.dataset_list_url()) + + +@login_required +@require_POST +def dataset_style(request, layername): + layer = _resolve_dataset(request, layername, "base.change_resourcebase", _PERMISSION_MSG_MODIFY) + + style_name = request.POST.get("defaultStyle") + + # would be nice to implement + # better handling of default style switching + # in layer model or deeper (gsconfig.py, REST API) + + old_default = layer.default_style + if old_default.name == style_name: + return HttpResponse(f"Default style for {layer.name} remains {style_name}", status=200) + + # This code assumes without checking + # that the new default style name is included + # in the list of possible styles. + + new_style = next(style for style in layer.styles if style.name == style_name) + + # Does this change this in geoserver?? + layer.default_style = new_style + layer.styles = [s for s in layer.styles if s.name != style_name] + [old_default] + layer.save(notify=True) + + # Invalidate GeoWebCache for the updated resource + try: + _stylefilterparams_geowebcache_dataset(layer.alternate) + _invalidate_geowebcache_dataset(layer.alternate) + except Exception: + pass + + return HttpResponse(f"Default style for {layer.name} changed to {style_name}", status=200) + + +def style_change_check(request, path, style_name=None, access_token=None): + """ + If the layer has not change_dataset_style permission, return a status of + 401 (unauthorized) + """ + # a new style is created with a POST and then a PUT, + # a style is updated with a PUT + # a layer is updated with a style with a PUT + # in both case we need to check permissions here + # for PUT path is /gs/rest/styles/san_andres_y_providencia_water_a452004b.xml + # or /ge/rest/layers/geonode:san_andres_y_providencia_coastline.json + # for POST path is /gs/rest/styles + # we will suppose that a user can create a new style only if he is an + # authenticated (we need to discuss about it) + authorized = True + if request.method in ("PUT", "POST"): + if not request.user.is_authenticated and not access_token: + authorized = False + elif re.match(r"^.*(? 0: + if ws: + # Lets try + # http://localhost:8080/geoserver/rest/workspaces//styles/ +{% endblock head %} + +{% block sidebar %} +
      +
      +

      {% trans "Edit Members for" %} {{ group.title }}

      +

      {% trans "Current Members" %}

      +
      +
      + +
      +
      +
        + {% for member in members %} +

        + {{ member.user.username }} + {% if member.user.email %} + + {% endif %} + | + {% if member.role == 'manager' %} + {% trans 'Manager' %} + | + {% endif %} +
        + {% csrf_token %} + +
        + {% if member.role == 'member' %} + | +
        + {% csrf_token %} + +
        + {% endif %} + {% if member.role == 'manager' %} + | +
        + {% csrf_token %} + +
        + {% endif %} +

        +

        + {% trans 'Role' %}: {{ member.role }} +
        +
        + {% endfor %} +
      +
      +
      +
        + {% for member in members %} + {% if member.role == 'manager' %} +

        + {{ member.user.username }} + {% if member.user.email %} + + {% endif %} + | + {% if member.role == 'manager' %} + {% trans 'Manager' %} + | + {% endif %} +
        + {% csrf_token %} + +
        + | +
        + {% csrf_token %} + +
        +

        +

        + {% trans 'Role' %}: {{ member.role }} +
        +
        + {% endif %} + {% endfor %} +
      +
      +
      +
        + {% for member in members %} + {% if member.role == 'member' %} +

        + {{ member.user.username }} + {% if member.user.email %} + + {% endif %} + | + {% if member.role == 'manager' %} + {% trans 'Manager' %} + | + {% endif %} +
        + {% csrf_token %} + +
        + | +
        + {% csrf_token %} + +
        +

        +

        + {% trans 'Role' %}: {{ member.role }} +
        +
        + {% endif %} + {% endfor %} +
      +
      +
      +
      +
      + {% if member_form %} +
      +

      {% trans "Add new members" %}

      +
      + {% csrf_token %} +
      + {{ member_form|as_bootstrap }}

      +
      + +
      +
      + {% endif %} +
      +
      +{% endblock %} + +{% block extra_script %} +{{ block.super }} +{% endblock extra_script %} diff --git a/geonode/groups/templates/groups/group_remove.html b/geonode/groups/templates/groups/group_remove.html new file mode 100644 index 0000000..6b909ec --- /dev/null +++ b/geonode/groups/templates/groups/group_remove.html @@ -0,0 +1,25 @@ +{% extends "groups/group_base.html" %} +{% load i18n %} + +{% block title %}{{ group.title }} — {{ block.super }}{% endblock %} +{% block body_class %}groups remove{% endblock %} + +{% block body %} +
      +
      +
      +

      + {% blocktrans with group.title as group_title %} + Are you sure you want to remove the group: {{ group_title }}? + {% endblocktrans %} +

      + {% csrf_token %} + +
      + + {% trans "No, don't remove it" %} +
      +
      +
      +
      +{% endblock %} diff --git a/geonode/groups/templates/groups/group_remove_maps_data.html b/geonode/groups/templates/groups/group_remove_maps_data.html new file mode 100644 index 0000000..a0c15e3 --- /dev/null +++ b/geonode/groups/templates/groups/group_remove_maps_data.html @@ -0,0 +1,18 @@ +{% extends "groups/group_base.html" %} +{% load i18n %} + +{% block sidebar %} +
      +

      {% trans "Remove Maps/Data from Group" %}

      +
      + {% csrf_token %} +
        + {{ map_form.as_ul }} +
      +
        + {{ dataset_form.as_ul }} +
      + +
      +
      +{% endblock %} diff --git a/geonode/groups/templates/groups/group_update.html b/geonode/groups/templates/groups/group_update.html new file mode 100644 index 0000000..e8eb37e --- /dev/null +++ b/geonode/groups/templates/groups/group_update.html @@ -0,0 +1,34 @@ +{% extends "groups/group_base.html" %} +{% load bootstrap_tags %} +{% load i18n %} + +{% block head_title %}{% trans "Create Group" %}{% endblock %} +{% block body_class %}groups update{% endblock %} + +{% block body %} +
      +
      +

      {% trans "Update Group" %}

      + {% if errors %} +
      + {% for error in errors %} +
      {{ error }}
      + {% endfor %} +
      + {% endif %} +
      +
      + {% csrf_token %} +
      + {{ form|as_bootstrap }} +
      + + +
      +
      +
      +
      +
      +
      + +{% endblock %} diff --git a/geonode/groups/templates/groups/groupcategory_detail.html b/geonode/groups/templates/groups/groupcategory_detail.html new file mode 100644 index 0000000..2e52cc8 --- /dev/null +++ b/geonode/groups/templates/groups/groupcategory_detail.html @@ -0,0 +1,51 @@ +{% extends "groups/group_detail.html" %} +{% load i18n %} + + +{% block title %}{{ object.name|default:object.slug }} — {{ block.super.super }}{% endblock %} + +{% block body_outer %} +
      +

      {{ object.name|default:object.slug }}

      +
      +
      +
      +
      +
      +

      {{ object.description|default_if_none:'' }}

      + +
      +
      +
      + +
      + +
      +
      +
      +
      +

      {% trans "Groups" %}

      +
      + {% include 'groups/_group_profile_list_item.html' %} +
      +
      + {% include 'search/_pagination.html' %} +
      +{% endblock %} + + +{% block extra_script %} +{{ block.super.super }} + + {% with include_spatial='false' %} + {% include 'search/search_scripts.html' %} + {% endwith %} +{% endblock extra_script %} + + diff --git a/geonode/groups/templates/groups/groupcategory_form.html b/geonode/groups/templates/groups/groupcategory_form.html new file mode 100644 index 0000000..2ea9d59 --- /dev/null +++ b/geonode/groups/templates/groups/groupcategory_form.html @@ -0,0 +1,6 @@ +{% extends "groups/group_create.html" %} +{% load i18n %} + +{% block head_title %}{% trans "Create Group Category" %}{% endblock %} + +{% block body_page_title %}{% trans "Create a Group Category" %}{% endblock %} diff --git a/geonode/groups/templates/groups/groupcategory_update_form.html b/geonode/groups/templates/groups/groupcategory_update_form.html new file mode 100644 index 0000000..c6c56e2 --- /dev/null +++ b/geonode/groups/templates/groups/groupcategory_update_form.html @@ -0,0 +1,8 @@ +{% extends "groups/group_create.html" %} +{% load i18n %} + +{% block head_title %}{% trans "Edit Group Category" %} {{ object.name }}{% endblock %} + +{% block body_page_title %}{% trans "Edit Group Category" %} {{ object.name }}{% endblock %} + +{% block form_button %}{% trans "Save" %}{% endblock %} diff --git a/geonode/groups/templatetags/__init__.py b/geonode/groups/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/groups/templatetags/groups_tags.py b/geonode/groups/templatetags/groups_tags.py new file mode 100644 index 0000000..124aed5 --- /dev/null +++ b/geonode/groups/templatetags/groups_tags.py @@ -0,0 +1,51 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django import template +from django.contrib.staticfiles.storage import staticfiles_storage + +register = template.Library() + + +@register.simple_tag +def group_profile_image(group_profile, css_classes="", size=None): + """Returns an HTML img tag with the input group_profiles's logo. + + If the group profile does not have an associated logo, a stock image is + used. + + """ + + if isinstance(css_classes, str): + class_attr = f'class="{css_classes}" ' + else: + try: + class_attr = f'class="{(" ".join(str(i) for i in css_classes))}" ' + except Exception: + class_attr = "" + if size is not None: + style_attr = f'style="width: {size}px; height: {size}px" ' + else: + style_attr = "" + + if group_profile.logo.name: + url = group_profile.logo_url + else: + url = staticfiles_storage.url("geonode/img/default-avatar.jpg") + img_tag = f'{group_profile.title}' + return img_tag diff --git a/geonode/groups/tests.py b/geonode/groups/tests.py new file mode 100644 index 0000000..3b84941 --- /dev/null +++ b/geonode/groups/tests.py @@ -0,0 +1,800 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import json +import logging + +from django.urls import reverse +from django.conf import settings +from django.test import override_settings +from django.contrib.auth.models import Group +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile + +from geonode.maps.models import Map +from geonode.layers.models import Dataset +from geonode.documents.models import Document +from guardian.shortcuts import get_anonymous_user +from geonode.security.views import _perms_info_json +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.groups.conf import settings as groups_settings + +from geonode.groups.models import GroupProfile, GroupMember, GroupCategory + +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset + +logger = logging.getLogger(__name__) + + +def _log(msg, *args): + logger.debug(msg, *args) + + +class GroupsSmokeTest(GeoNodeBaseTestSupport): + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + + self.norman = get_user_model().objects.get(username="norman") + self.norman.groups.add(Group.objects.get(name="anonymous")) + self.test_user = get_user_model().objects.get(username="test_user") + self.test_user.groups.add(Group.objects.get(name="anonymous")) + self.bar = GroupProfile.objects.get(slug="bar") + self.anonymous_user = get_anonymous_user() + + c1 = GroupCategory.objects.create(name="test #1 category") + g = GroupProfile.objects.create(slug="test", title="test") + g.categories.add(c1) + g.save() + User = get_user_model() + u = User.objects.create(username="test") + u.set_password("test") + u.save() + + """ + Basic checks to make sure pages load, etc. + """ + + def test_registered_group_exists(self): + """ + Ensures that a default group and grouprofile 'registered-users' has been + created at initialization time. + """ + group = Group.objects.filter(name=groups_settings.REGISTERED_MEMBERS_GROUP_NAME).first() + self.assertTrue(group) + + def test_users_group_list_view(self): + """ + 1. Ensures that a superuser can see the whole group list. + + 2. Ensures that a user can see only public/public-invite groups. + + 3. Ensures that a user belonging to a private group, can see it. + """ + bobby = get_user_model().objects.get(username="bobby") + + public_group, _public_created = GroupProfile.objects.get_or_create( + slug="public_group", title="public_group", access="public" + ) + private_group, _private_created = GroupProfile.objects.get_or_create( + slug="private_group", title="private_group", access="private" + ) + + private_group.join(bobby) + data = {"query": "p", "page": 1, "pageSize": 9} + + # Anonymous + """ + '{ + "users": [], "count": 0, + "groups": [ + {"name": "public_group", "title": "public_group"}] + }' + """ + response = self.client.post(reverse("account_ajax_lookup"), data) + self.assertEqual(response.status_code, 200) + content = json.loads(response.content) + logger.debug(f"Anonymous --> {content}") + self.assertEqual(len(content["groups"]), 1) + self.assertEqual(content["groups"][0]["name"], "public_group") + response = self.client.get( + reverse( + "group_detail", + args=[ + "public_group", + ], + ) + ) + self.assertEqual(response.status_code, 200) + response = self.client.get( + reverse( + "group_detail", + args=[ + "private_group", + ], + ) + ) + self.assertEqual(response.status_code, 404) + + # Admin + """ + '{ + "users": [], "count": 0, + "groups": [ + {"name": "public_group", "title": "public_group"}, + {"name": "private_group", "title": "private_group"}] + }' + """ + self.assertTrue(self.client.login(username="admin", password="admin")) + response = self.client.post(reverse("account_ajax_lookup"), data) + self.assertEqual(response.status_code, 200) + content = json.loads(response.content) + logger.debug(f"admin --> {content}") + self.assertEqual(len(content["groups"]), 2) + self.assertEqual(content["groups"][0]["name"], "public_group") + self.assertEqual(content["groups"][1]["name"], "private_group") + response = self.client.get( + reverse( + "group_detail", + args=[ + "public_group", + ], + ) + ) + self.assertEqual(response.status_code, 200) + response = self.client.get( + reverse( + "group_detail", + args=[ + "private_group", + ], + ) + ) + self.assertEqual(response.status_code, 200) + + # Bobby + """ + '{ + "users": [], "count": 0, + "groups": [ + {"name": "public_group", "title": "public_group"}, + {"name": "private_group", "title": "private_group"}] + }' + """ + self.assertTrue(self.client.login(username="bobby", password="bob")) + response = self.client.post(reverse("account_ajax_lookup"), data) + self.assertEqual(response.status_code, 200) + content = json.loads(response.content) + logger.debug(f"bobby --> {content}") + self.assertEqual(len(content["groups"]), 2) + self.assertEqual(content["groups"][0]["name"], "public_group") + self.assertEqual(content["groups"][1]["name"], "private_group") + response = self.client.get( + reverse( + "group_detail", + args=[ + "public_group", + ], + ) + ) + self.assertEqual(response.status_code, 200) + response = self.client.get( + reverse( + "group_detail", + args=[ + "private_group", + ], + ) + ) + self.assertEqual(response.status_code, 200) + + # Norman + """ + '{ + "users": [], "count": 0, + "groups": [ + {"name": "public_group", "title": "public_group"}] + }' + """ + self.assertTrue(self.client.login(username="norman", password="norman")) + response = self.client.post(reverse("account_ajax_lookup"), data) + self.assertEqual(response.status_code, 200) + content = json.loads(response.content) + logger.debug(f"norman --> {content}") + self.assertEqual(len(content["groups"]), 1) + self.assertEqual(content["groups"][0]["name"], "public_group") + response = self.client.get( + reverse( + "group_detail", + args=[ + "public_group", + ], + ) + ) + self.assertEqual(response.status_code, 200) + response = self.client.get( + reverse( + "group_detail", + args=[ + "private_group", + ], + ) + ) + self.assertEqual(response.status_code, 404) + + if _public_created: + public_group.delete() + self.assertFalse(GroupProfile.objects.filter(slug="public_group").exists()) + if _private_created: + private_group.delete() + self.assertFalse(GroupProfile.objects.filter(slug="private_group").exists()) + + def test_group_permissions_extend_to_user(self): + """ + Ensures that when a user is in a group, the group permissions + extend to the user. + """ + layer = Dataset.objects.first() + # Set the default permissions + layer.set_default_permissions() + + # Test that the anonymous user can read + self.assertTrue(self.anonymous_user.has_perm("view_resourcebase", layer.get_self_resource())) + + # Test that the default perms give Norman view permissions but not + # write permissions + self.assertTrue(self.norman.has_perm("view_resourcebase", layer.get_self_resource())) + self.assertFalse(self.norman.has_perm("change_resourcebase", layer.get_self_resource())) + + # Make sure Norman is not in the bar group. + self.assertFalse(self.bar.user_is_member(self.norman)) + + # Add norman to the bar group. + self.bar.join(self.norman) + + # Ensure Norman is in the bar group. + self.assertTrue(self.bar.user_is_member(self.norman)) + + # Give the bar group permissions to change the layer. + permissions = {"groups": {"bar": ["view_resourcebase", "change_resourcebase"]}} + layer.set_permissions(permissions) + + self.assertTrue(self.norman.has_perm("view_resourcebase", layer.get_self_resource())) + # check that now norman can change the layer + self.assertTrue(self.norman.has_perm("change_resourcebase", layer.get_self_resource())) + + # Test adding a new user to the group after setting permissions on the layer. + # Make sure Test User is not in the bar group. + self.assertFalse(self.bar.user_is_member(self.test_user)) + + self.assertFalse(self.test_user.has_perm("change_resourcebase", layer.get_self_resource())) + + self.bar.join(self.test_user) + + self.assertTrue(self.test_user.has_perm("change_resourcebase", layer.get_self_resource())) + + def test_group_resource(self): + """ + Tests the resources method on a Group object. + """ + + layer = Dataset.objects.first() + map = Map.objects.first() + + perm_spec = {"groups": {"bar": ["change_resourcebase"]}} + # Give the self.bar group write perms on the layer + layer.set_permissions(perm_spec) + map.set_permissions(perm_spec) + + # Ensure the layer is returned in the group's resources + self.assertTrue(layer.get_self_resource() in self.bar.resources()) + self.assertTrue(map.get_self_resource() in self.bar.resources()) + + # Test the resource filter + self.assertTrue(layer.get_self_resource() in self.bar.resources(resource_type="dataset")) + self.assertTrue(map.get_self_resource() not in self.bar.resources(resource_type="dataset")) + + # Revoke permissions on the layer from the self.bar group + layer.set_permissions("{}") + + # Ensure the layer is no longer returned in the groups resources + self.assertFalse(layer.get_self_resource() in self.bar.resources()) + + def test_perms_info(self): + """ + Tests the perms_info function (which passes permissions to the response context). + """ + # Add test to test perms being sent to the front end. + layer = Dataset.objects.first() + layer.set_default_permissions() + perms_info = layer.get_all_level_info() + + # Ensure there is only one group 'anonymous' by default + self.assertEqual(len(perms_info["groups"].keys()), 1) + + # Add the foo group to the layer object groups + layer.set_permissions({"groups": {"bar": ["view_resourcebase"]}}) + + perms_info = _perms_info_json(layer) + # Ensure foo is in the perms_info output + self.assertCountEqual(json.loads(perms_info)["groups"], {"bar": ["view_resourcebase"]}) + + def test_resource_permissions(self): + """ + Tests that the client can get and set group permissions through the test_resource_permissions view. + """ + + self.assertTrue(self.client.login(username="admin", password="admin")) + + layer = Dataset.objects.first() + document = Document.objects.first() + map_obj = Map.objects.first() + layer.set_default_permissions() + document.set_default_permissions() + map_obj.set_default_permissions() + + objects = layer, document, map_obj + + for obj in objects: + response = self.client.get(reverse("resource_permissions", kwargs=dict(resource_id=obj.id))) + self.assertEqual(response.status_code, 200) + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + js = json.loads(content) + permissions = js.get("permissions", dict()) + + if isinstance(permissions, str): + permissions = json.loads(permissions) + + # Ensure the groups value is empty by default + expected_permissions = {} + if settings.DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION: + expected_permissions.setdefault("anonymous", []).append("download_resourcebase") + if settings.DEFAULT_ANONYMOUS_VIEW_PERMISSION: + expected_permissions.setdefault("anonymous", []).append("view_resourcebase") + + self.assertCountEqual(permissions.get("groups"), expected_permissions) + + permissions = {"groups": {"bar": ["change_resourcebase"]}, "users": {"admin": ["change_resourcebase"]}} + + # Give the bar group permissions + response = self.client.post( + reverse("resource_permissions", kwargs=dict(resource_id=obj.id)), + data=json.dumps(permissions), + content_type="application/json", + ) + + self.assertEqual(response.status_code, 200) + + response = self.client.get(reverse("resource_permissions", kwargs=dict(resource_id=obj.id))) + + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + js = json.loads(content) + permissions = js.get("permissions", dict()) + + if isinstance(permissions, str): + permissions = json.loads(permissions) + + # Make sure the bar group now has write permissions + self.assertCountEqual(permissions["groups"], {"bar": ["change_resourcebase"]}) + + # Remove group permissions + permissions = {"users": {"admin": ["change_resourcebase"]}} + + # Update the object's permissions to remove the bar group + response = self.client.post( + reverse("resource_permissions", kwargs=dict(resource_id=obj.id)), + data=json.dumps(permissions), + content_type="application/json", + ) + + self.assertEqual(response.status_code, 200) + + response = self.client.get(reverse("resource_permissions", kwargs=dict(resource_id=obj.id))) + + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + js = json.loads(content) + permissions = js.get("permissions", dict()) + + if isinstance(permissions, str): + permissions = json.loads(permissions) + + # Assert the bar group no longer has permissions + self.assertCountEqual(permissions["groups"], {}) + + def test_create_new_group(self): + """ + Tests creating a group through the group_create route. + """ + + d = dict(title="TestGroup", description="This is a test group.", access="public", keywords="testing, groups") + + self.client.login(username="admin", password="admin") + response = self.client.post(reverse("group_create"), data=d) + # successful POSTS will redirect to the group's detail view. + self.assertEqual(response.status_code, 302) + self.assertTrue(GroupProfile.objects.get(title="TestGroup")) + + def test_delete_group_view(self): + """ + Tests deleting a group through the group_delete route. + """ + + # Ensure the group exists + self.assertTrue(GroupProfile.objects.get(id=self.bar.id)) + + self.client.login(username="admin", password="admin") + + # Delete the group + response = self.client.post(reverse("group_remove", args=[self.bar.slug])) + + # successful POSTS will redirect to the group list view. + self.assertEqual(response.status_code, 302) + self.assertFalse(GroupProfile.objects.filter(id=self.bar.id).exists()) + + def test_delete_group_view_no_perms(self): + """ + Tests deleting a group through the group_delete with a non-manager. + """ + + # Ensure the group exists + self.assertTrue(GroupProfile.objects.get(id=self.bar.id)) + + self.client.login(username="norman", password="norman") + + # Delete the group + response = self.client.post(reverse("group_remove", args=[self.bar.slug])) + + self.assertEqual(response.status_code, 403) + + # Ensure the group still exists + self.assertTrue(GroupProfile.objects.get(id=self.bar.id)) + + def test_groupmember_manager(self): + """ + Tests the get_managers method. + """ + norman = get_user_model().objects.get(username="norman") + admin = get_user_model().objects.get(username="admin") + + # Make sure norman is not a user + self.assertFalse(self.bar.user_is_member(norman)) + + # Add norman to the self.bar group + self.bar.join(norman) + + # Ensure norman is now a member + self.assertTrue(self.bar.user_is_member(norman)) + + # Ensure norman is not in the managers queryset + self.assertTrue(norman not in self.bar.get_managers()) + + # Ensure admin is in the managers queryset + self.bar.join(admin, role=GroupMember.MANAGER) + self.assertTrue(admin in self.bar.get_managers()) + + def test_public_pages_render(self): + """ + Verify pages that do not require login load without internal error + """ + + response = self.client.get("/groups/") + self.assertEqual(response.status_code, 200) + + response = self.client.get("/groups/group/bar/") + self.assertEqual(response.status_code, 200) + + response = self.client.get("/groups/group/bar/members/") + self.assertEqual(response.status_code, 200) + + # 302 for auth failure since we redirect to login page + response = self.client.get("/groups/create/") + self.assertTrue(response.status_code in (302, 403)) + + response = self.client.get("/groups/group/bar/update/") + self.assertEqual(response.status_code, 302) + + # # 405 - json endpoint, doesn't support GET + # response = self.client.get("/groups/group/bar/invite/") + # self.assertEqual(405, response.status_code) + + def test_protected_pages_render(self): + """ + Verify pages that require login load without internal error + """ + + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get("/groups/") + self.assertEqual(200, response.status_code) + + response = self.client.get("/groups/group/bar/") + self.assertEqual(200, response.status_code) + + response = self.client.get("/groups/group/bar/members/") + self.assertEqual(200, response.status_code) + + response = self.client.get("/groups/create/") + self.assertEqual(200, response.status_code) + + response = self.client.get("/groups/group/bar/update/") + self.assertEqual(200, response.status_code) + + # # 405 - json endpoint, doesn't support GET + # response = self.client.get("/groups/group/bar/invite/") + # self.assertEqual(405, response.status_code) + + """ + Tests membership logic in the geonode.groups models + """ + + def test_group_is_member(self): + """ + Tests checking group membership + """ + + anon = get_anonymous_user() + normal = get_user_model().objects.get(username="norman") + group = GroupProfile.objects.get(slug="bar") + + self.assertFalse(group.user_is_member(anon)) + self.assertFalse(group.user_is_member(normal)) + + def test_group_add_member(self): + """ + Tests adding a user to a group + """ + + anon = get_anonymous_user() + normal = get_user_model().objects.get(username="norman") + group = GroupProfile.objects.get(slug="bar") + group.join(normal) + self.assertTrue(group.user_is_member(normal)) + self.assertRaises(ValueError, lambda: group.join(anon)) + + def test_group_promote_demote_member(self): + """ + Tests promoting a member to manager, demoting to member + """ + + normal = get_user_model().objects.get(username="norman") + group = GroupProfile.objects.get(slug="bar") + group.join(normal) + self.assertFalse(group.user_is_role(normal, "manager")) + GroupMember.objects.get(group=group, user=normal).promote() + self.assertTrue(group.user_is_role(normal, "manager")) + GroupMember.objects.get(group=group, user=normal).demote() + self.assertFalse(group.user_is_role(normal, "manager")) + + def test_profile_is_member_of_group(self): + """ + Tests profile is_member_of_group property + """ + + normal = get_user_model().objects.get(username="norman") + group = GroupProfile.objects.get(slug="bar") + self.assertFalse(normal.is_member_of_group(group.slug)) + + group.join(normal) + self.assertTrue(normal.is_member_of_group(group.slug)) + + def test_group_remove_member(self): + """ + Tests removing a user from a group + """ + + normal = get_user_model().objects.get(username="norman") + group = GroupProfile.objects.get(slug="bar") + group.join(normal) + self.assertTrue(group.user_is_member(normal)) + group.leave(normal) + self.assertFalse(group.user_is_member(normal)) + + @override_settings(MEDIA_ROOT="/tmp/geonode_tests") + def test_group_logo_is_present_on_list_view(self): + """Verify that a group's logo is rendered on list view.""" + with self.settings(API_LOCKDOWN=False): + test_profile, _ = GroupProfile.objects.update_or_create( + slug="test", + defaults=dict( + description="test", access="public", logo=SimpleUploadedFile("dummy-file.jpg", b"dummy contents") + ), + ) + + response = self.client.get( + reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "groups"}) + ) + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + response_payload = json.loads(content) + returned = response_payload["objects"] + group_profile = [g["group_profile"] for g in returned if g["group_profile"]["title"] == test_profile.title][ + 0 + ] + self.assertEqual(200, response.status_code) + self.assertEqual(group_profile["logo"], test_profile.logo.url) + + def test_group_logo_is_not_present_on_list_view(self): + """ + Verify that no logo exists in list view when a group doesn't have one. + """ + + with self.settings(API_LOCKDOWN=False): + test_profile, _ = GroupProfile.objects.update_or_create( + slug="test", defaults=dict(title="test", description="test", access="public") + ) + + response = self.client.get( + reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "groups"}) + ) + content = response.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + response_payload = json.loads(content) + returned = response_payload["objects"] + group_profile = [g["group_profile"] for g in returned if g["group_profile"]["title"] == test_profile.title][ + 0 + ] + self.assertEqual(200, response.status_code) + self.assertIsNone(group_profile["logo"]) + + def test_group_activity_pages_render(self): + """ + Verify Activity List pages + """ + + self.assertTrue(self.client.login(username="admin", password="admin")) + + response = self.client.get("/groups/") + self.assertEqual(200, response.status_code) + + response = self.client.get("/groups/group/bar/activity/") + self.assertEqual(200, response.status_code) + + self.assertContains(response, "Datasets", count=3, status_code=200, msg_prefix="", html=False) + self.assertContains(response, "Maps", count=3, status_code=200, msg_prefix="", html=False) + self.assertContains(response, "Documents", count=3, status_code=200, msg_prefix="", html=False) + self.assertContains( + response, 'CA', count=0, status_code=200, msg_prefix="", html=False + ) + self.assertContains(response, "uploaded", count=0, status_code=200, msg_prefix="", html=False) + dataset = create_single_dataset("single_point.shp") + try: + # Add test to test perms being sent to the front end. + dataset.set_default_permissions() + perms_info = dataset.get_all_level_info() + + # Ensure there is only one group 'anonymous' by default + self.assertEqual(len(perms_info["groups"].keys()), 1) + + # Add the foo group to the dataset object groups + perms_info["groups"]["bar"] = ["view_resourcebase"] + dataset.set_permissions(perms_info) + + perms_info = _perms_info_json(dataset) + # Ensure foo is in the perms_info output + self.assertCountEqual(json.loads(perms_info)["groups"]["bar"], ["view_resourcebase"]) + + dataset.group = self.bar.group + dataset.save() + + response = self.client.get("/groups/group/bar/activity/") + self.assertEqual(200, response.status_code) + _log(response) + self.assertContains( + response, + f'geonode:single_point.shp', + count=2, + status_code=200, + msg_prefix="", + html=False, + ) + self.assertContains(response, "uploaded", count=2, status_code=200, msg_prefix="", html=False) + finally: + dataset.set_default_permissions() + dataset.group = None + dataset.save() + + """ + Group Categories tests + """ + + def test_api(self): + api_url = "/api/groupcategory/" + + self.client.login(username="test", password="test") # login necessary because settings.API_LOCKDOWN=True + r = self.client.get(api_url) + self.assertEqual(r.status_code, 200) + content = r.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + data = json.loads(content) + self.assertEqual(data["meta"]["total_count"], GroupCategory.objects.all().count()) + + # check if we have non-empty group category + self.assertTrue(GroupCategory.objects.filter(groups__isnull=False).exists()) + + for item in data["objects"]: + self.assertTrue(GroupCategory.objects.filter(slug=item["slug"]).count() == 1) + g = GroupCategory.objects.get(slug=item["slug"]) + self.assertEqual(item["member_count"], g.groups.all().count()) + + self.client.logout() + r = self.client.get(api_url) + self.assertEqual(r.status_code, 200) + content = r.content + if isinstance(content, bytes): + content = content.decode("UTF-8") + data = json.loads(content) + self.assertEqual(data["meta"]["total_count"], 1) + + # check if we have non-empty group category + self.assertTrue(GroupCategory.objects.filter(groups__isnull=False).exists()) + + for item in data["objects"]: + self.assertTrue(GroupCategory.objects.filter(slug=item["slug"]).count() == 1) + g = GroupCategory.objects.get(slug=item["slug"]) + self.assertEqual(item["member_count"], 1) + + def test_group_categories_list(self): + view_url = reverse("group_category_list") + r = self.client.get(view_url) + self.assertEqual(r.status_code, 200) + + def test_group_categories_add(self): + view_url = reverse("group_category_create") + # Test that the view is protected to anonymous users + r = self.client.get(view_url) + self.assertTrue(r.status_code in (302, 403)) + + # Test that the view is protected to non-admin users + self.client.login(username="test", password="test") + r = self.client.post(view_url) + self.assertTrue(r.status_code in (401, 403)) + + # Test that the view is accessible to administrators + self.client.login(username="admin", password="admin") + r = self.client.get(view_url) + self.assertEqual(r.status_code, 200) + + # Create e new category + category = "test #3 category" + r = self.client.post(view_url, {"name": category}) + + self.assertEqual(r.status_code, 302) + q = GroupCategory.objects.filter(name=category) + self.assertEqual(q.count(), 1) + self.assertTrue(q.get().slug) diff --git a/geonode/groups/translation.py b/geonode/groups/translation.py new file mode 100644 index 0000000..c14252d --- /dev/null +++ b/geonode/groups/translation.py @@ -0,0 +1,37 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from modeltranslation.translator import translator, TranslationOptions + +from geonode.groups.models import GroupCategory, GroupProfile + + +class GroupCategoryTranslationOptions(TranslationOptions): + fields = ("name",) + + +class GroupProfileTranslationOptions(TranslationOptions): + fields = ( + "title", + "description", + ) + + +translator.register(GroupCategory, GroupCategoryTranslationOptions) +translator.register(GroupProfile, GroupProfileTranslationOptions) diff --git a/geonode/groups/urls.py b/geonode/groups/urls.py new file mode 100644 index 0000000..7f485ad --- /dev/null +++ b/geonode/groups/urls.py @@ -0,0 +1,60 @@ +# +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# + +from django.urls import re_path +from django.views.generic import TemplateView + +from .views import GroupDetailView, GroupActivityView, SetGroupDatasetPermission +from . import views + +urlpatterns = [ # 'geonode.groups.views', + re_path(r"^$", TemplateView.as_view(template_name="groups/group_list.html"), name="group_list"), + re_path( + r"^categories/$", TemplateView.as_view(template_name="groups/category_list.html"), name="group_category_list" + ), + re_path(r"^categories/_create/$", views.group_category_create, name="group_category_create"), + re_path(r"^categories/(?P[-\w]+)/$", views.group_category_detail, name="group_category_detail"), + re_path(r"^categories/(?P[-\w]+)/update/$", views.group_category_update, name="group_category_update"), + re_path(r"^create/$", views.group_create, name="group_create"), + re_path(r"^group/(?P[-\w]+)/$", GroupDetailView.as_view(), name="group_detail"), + re_path(r"^group/(?P[-\w]+)/update/$", views.group_update, name="group_update"), + re_path(r"^group/(?P[-\w]+)/members/$", views.group_members, name="group_members"), + re_path(r"^group/(?P[-\w]+)/members_add/$", views.group_members_add, name="group_members_add"), + re_path( + r"^group/(?P[-\w]+)/member_remove/(?P.+)$", + views.group_member_remove, + name="group_member_remove", + ), + re_path( + r"^group/(?P[-\w]+)/member_promote/(?P.+)$", + views.group_member_promote, + name="group_member_promote", + ), + re_path( + r"^group/(?P[-\w]+)/member_demote/(?P.+)$", + views.group_member_demote, + name="group_member_demote", + ), + re_path(r"^group/(?P[-\w]+)/remove/$", views.group_remove, name="group_remove"), + re_path(r"^group/(?P[-\w]+)/join/$", views.group_join, name="group_join"), + re_path(r"^group/(?P[-\w]+)/activity/$", GroupActivityView.as_view(), name="group_activity"), + re_path(r"^autocomplete/$", views.GroupProfileAutocomplete.as_view(), name="autocomplete_groups"), + re_path(r"^autocomplete_category/$", views.GroupCategoryAutocomplete.as_view(), name="autocomplete_category"), + re_path(r"^dataset/permission/$", SetGroupDatasetPermission.as_view(), name="set_group_dataset_permissions"), +] diff --git a/geonode/groups/views.py b/geonode/groups/views.py new file mode 100644 index 0000000..1244f0f --- /dev/null +++ b/geonode/groups/views.py @@ -0,0 +1,324 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.views import View +from geonode.base.views import user_and_group_permission +import logging + +from actstream.models import Action +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required +from django.urls import reverse +from django.http import Http404, HttpResponseForbidden, HttpResponseNotAllowed, HttpResponseRedirect +from django.contrib import messages +from django.shortcuts import get_object_or_404, redirect, render +from django.views.decorators.http import require_POST +from django.views.generic import ListView, CreateView +from django.views.generic.edit import UpdateView +from django.views.generic.detail import DetailView +from django.db.models import Q + +from geonode.decorators import view_decorator, superuser_only +from geonode.base.views import SimpleSelect2View + +from dal import autocomplete + +from . import forms +from . import models +from .models import GroupMember + +logger = logging.getLogger(__name__) + + +class SetGroupDatasetPermission(View): + def get(self, request): + return user_and_group_permission(request, "groupprofile") + + def post(self, request): + return user_and_group_permission(request, "groupprofile") + + +@view_decorator(superuser_only, subclass=True) +class GroupCategoryCreateView(CreateView): + model = models.GroupCategory + fields = ["name", "description"] + + +class GroupCategoryDetailView(DetailView): + model = models.GroupCategory + + +class GroupCategoryUpdateView(UpdateView): + model = models.GroupCategory + fields = ["name", "description"] + template_name_suffix = "_update_form" + + +group_category_create = GroupCategoryCreateView.as_view() +group_category_detail = GroupCategoryDetailView.as_view() +group_category_update = GroupCategoryUpdateView.as_view() + + +@superuser_only +def group_create(request): + if request.method == "POST": + form = forms.GroupForm(request.POST, request.FILES) + if form.is_valid(): + group = form.save(commit=False) + group.save() + form.save_m2m() + group.join(request.user, role="manager") + return HttpResponseRedirect(reverse("group_detail", args=[group.slug])) + else: + form = forms.GroupForm() + + return render(request, "groups/group_create.html", context={"form": form}) + + +@login_required +def group_update(request, slug): + group = models.GroupProfile.objects.get(slug=slug) + if not group.user_is_role(request.user, role="manager"): + return HttpResponseForbidden() + + if request.method == "POST": + form = forms.GroupUpdateForm(request.POST, request.FILES, instance=group) + if form.is_valid(): + group = form.save(commit=False) + group.save() + form.save_m2m() + return HttpResponseRedirect(reverse("group_detail", args=[group.slug])) + else: + form = forms.GroupForm(instance=group) + + return render( + request, + "groups/group_update.html", + context={ + "form": form, + "group": group, + }, + ) + + +class GroupDetailView(ListView): + """ + Mixes a detail view (the group) with a ListView (the members). + """ + + model = get_user_model() + template_name = "groups/group_detail.html" + paginate_by = None + group = None + + def get_queryset(self): + return self.group.member_queryset() + + def get(self, request, *args, **kwargs): + self.group = get_object_or_404(models.GroupProfile, slug=kwargs.get("slug")) + if self.group.access == "private" and not self.group.user_is_member(request.user): + raise Http404 + return super().get(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["object"] = self.group + context["maps"] = self.group.resources(resource_type="map") + context["datasets"] = self.group.resources(resource_type="dataset") + context["documents"] = self.group.resources(resource_type="document") + context["is_member"] = self.group.user_is_member(self.request.user) + context["is_manager"] = self.group.user_is_role(self.request.user, "manager") + context["can_view"] = self.group.can_view(self.request.user) + return context + + +def group_members(request, slug): + group = get_object_or_404(models.GroupProfile, slug=slug) + if not group.can_view(request.user): + raise Http404() + is_manager = group.user_is_role(request.user, "manager") + return render( + request, + "groups/group_members.html", + context={ + "group": group, + "members": group.member_queryset(), + "member_form": forms.GroupMemberForm(request.user) if is_manager else None, + }, + ) + + +@require_POST +@login_required +def group_members_add(request, slug): + group = get_object_or_404(models.GroupProfile, slug=slug) + if not group.user_is_role(request.user, role="manager"): + return HttpResponseForbidden() + form = forms.GroupMemberForm(request.user, request.POST) + if form.is_valid(): + for user in form.cleaned_data["user_identifiers"]: + try: + group.join(user, role=GroupMember.MANAGER if form.cleaned_data["manager_role"] else GroupMember.MEMBER) + except Exception as e: + messages.add_message(request, messages.ERROR, e) + return redirect("group_members", slug=group.slug) + return redirect("group_detail", slug=group.slug) + + +@login_required +def group_member_remove(request, slug, username): + group = get_object_or_404(models.GroupProfile, slug=slug) + user = get_object_or_404(get_user_model(), username=username) + + if not group.user_is_role(request.user, role="manager"): + return HttpResponseForbidden() + else: + GroupMember.objects.get(group=group, user=user).delete() + return redirect("group_detail", slug=group.slug) + + +@login_required +def group_member_promote(request, slug, username): + group = get_object_or_404(models.GroupProfile, slug=slug) + user = get_object_or_404(get_user_model(), username=username) + + if not group.user_is_role(request.user, role="manager"): + return HttpResponseForbidden() + else: + GroupMember.objects.get(group=group, user=user).promote() + return redirect("group_members", slug=group.slug) + + +@login_required +def group_member_demote(request, slug, username): + group = get_object_or_404(models.GroupProfile, slug=slug) + user = get_object_or_404(get_user_model(), username=username) + + if not group.user_is_role(request.user, role="manager"): + return HttpResponseForbidden() + else: + GroupMember.objects.get(group=group, user=user).demote() + return redirect("group_members", slug=group.slug) + + +@require_POST +@login_required +def group_join(request, slug): + group = get_object_or_404(models.GroupProfile, slug=slug) + + if group.access == "private": + raise Http404() + + if group.user_is_member(request.user): + return redirect("group_detail", slug=group.slug) + else: + group.join(request.user, role="member") + return redirect("group_detail", slug=group.slug) + + +@login_required +def group_remove(request, slug): + group = get_object_or_404(models.GroupProfile, slug=slug) + if request.method == "GET": + return render(request, "groups/group_remove.html", context={"group": group}) + if request.method == "POST": + if not group.user_is_role(request.user, role="manager"): + return HttpResponseForbidden() + + group.delete() + return HttpResponseRedirect(reverse("group_list")) + else: + return HttpResponseNotAllowed() + + +class GroupActivityView(ListView): + """ + Returns recent group activity. + """ + + template_name = "groups/activity.html" + group = None + + def get_queryset(self): + if not self.group: + return None + else: + members = [(member.user.id) for member in self.group.member_queryset()] + return Action.objects.filter( + public=True, + actor_object_id__in=members, + )[:15] + + def get(self, request, *args, **kwargs): + self.group = None + group = get_object_or_404(models.GroupProfile, slug=kwargs.get("slug")) + + if not group.can_view(request.user): + raise Http404() + + self.group = group + + return super().get(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + def getKey(action): + return action.timestamp + + context = super().get_context_data(**kwargs) + context["group"] = self.group + # Additional Filtered Lists Below + action_list = [] + actions = Action.objects.filter(public=True, action_object_content_type__model="dataset") + context["action_list_datasets"] = [ + action for action in actions if action.action_object and action.action_object.group == self.group.group + ][:15] + action_list.extend(context["action_list_datasets"]) + actions = Action.objects.filter(public=True, action_object_content_type__model="map")[:15] + context["action_list_maps"] = [ + action for action in actions if action.action_object and action.action_object.group == self.group.group + ][:15] + action_list.extend(context["action_list_maps"]) + actions = Action.objects.filter(public=True, action_object_content_type__model="document")[:15] + context["action_list_documents"] = [ + action for action in actions if action.action_object and action.action_object.group == self.group.group + ][:15] + action_list.extend(context["action_list_documents"]) + context["action_list"] = sorted(action_list, key=getKey, reverse=True) + return context + + +class GroupProfileAutocomplete(autocomplete.Select2QuerySetView): + def get_queryset(self): + request = self.request + user = request.user + qs = models.GroupProfile.objects.all() + + if self.q: + qs = qs.filter(title__icontains=self.q) + + if not user.is_authenticated or user.is_anonymous: + return qs.exclude(access="private") + elif not user.is_superuser: + return qs.filter(Q(pk__in=user.group_list_all()) | ~Q(access="private")) + return qs + + +class GroupCategoryAutocomplete(SimpleSelect2View): + model = models.GroupCategory + filter_arg = "name__icontains" diff --git a/geonode/harvesting/__init__.py b/geonode/harvesting/__init__.py new file mode 100644 index 0000000..6b4db60 --- /dev/null +++ b/geonode/harvesting/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/harvesting/admin.py b/geonode/harvesting/admin.py new file mode 100644 index 0000000..6bcd222 --- /dev/null +++ b/geonode/harvesting/admin.py @@ -0,0 +1,411 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import logging + +from django.contrib import ( + admin, + messages, +) +from django.contrib.humanize.templatetags import humanize +from django.urls import reverse +from django.utils.html import ( + format_html, + mark_safe, +) +from django.utils.translation import gettext_lazy as _ + +from . import ( + forms, + models, +) + +logger = logging.getLogger(__name__) + + +@admin.register(models.Harvester) +class HarvesterAdmin(admin.ModelAdmin): + form = forms.HarvesterForm + + list_display = ( + "id", + "status", + "name", + "scheduling_enabled", + "remote_available", + "get_num_harvestable_resources", + "show_link_to_selected_harvestable_resources", + "show_link_to_latest_harvesting_session", + "show_link_to_latest_refresh_session", + "get_worker_specific_configuration", + "get_time_until_next_availability_update", + "get_time_until_next_refresh", + "get_time_until_next_harvesting", + ) + list_filter = ("status",) + + readonly_fields = ( + "status", + "remote_available", + "last_checked_availability", + "last_checked_harvestable_resources", + "last_check_harvestable_resources_message", + "num_harvestable_resources", + "show_link_to_selected_harvestable_resources", + "show_link_to_latest_harvesting_session", + "show_link_to_latest_refresh_session", + ) + + list_editable = ("scheduling_enabled",) + + actions = [ + "update_harvester_availability", + "initiate_update_harvestable_resources", + "initiate_abort_update_harvestable_resources", + "initiate_perform_harvesting", + "initiate_abort_perform_harvesting", + "reset_harvester_status", + ] + + def save_model(self, request, harvester: models.Harvester, form, change): + super().save_model(request, harvester, form, change) + if _worker_config_changed(form): + self.message_user( + request, + ( + "Harvester worker specific configuration has been changed. " + "You should update the list of this harvester's harvestable " + "resources now in order to ensure consistency." + ), + level=messages.WARNING, + ) + + def get_form(self, request, obj=None, change=False, **kwargs): + form = super().get_form(request, obj, change, **kwargs) + form.base_fields["default_owner"].initial = request.user + return form + + @admin.action(description="Check availability of selected harvesters") + def update_harvester_availability(self, request, queryset): + updated_harvesters = [] + non_available_harvesters = [] + for harvester in queryset: + available = harvester.update_availability() + updated_harvesters.append(harvester) + if not available: + non_available_harvesters.append(harvester) + self.message_user(request, f"Updated availability for harvesters: {updated_harvesters}") + if len(non_available_harvesters) > 0: + self.message_user( + request, + (f"The following harvesters are not " f"available: {non_available_harvesters}"), + messages.WARNING, + ) + + @admin.action(description="Update harvestable resources for selected harvesters") + def initiate_update_harvestable_resources(self, request, queryset): + being_updated = [] + for harvester in queryset: + try: + if harvester.update_availability(): + harvester.initiate_update_harvestable_resources() + being_updated.append(harvester) + else: + raise RuntimeError(f"Harvester {harvester!r} is not available") + except RuntimeError as exc: + self.message_user(request, str(exc), level=messages.ERROR) + if len(being_updated) > 0: + message = ( + f"Updating harvestable resources asynchronously for {being_updated}. " + f"This operation can take a while to complete. Check the harvesters' " + f"status for when it becomes `ready` or inspect its latest refresh " + f"session and monitor the reported progress" + ) + else: + message = _("No ready harvesters have been selected, skipping...") + self.message_user(request, message) + + @admin.action(description="Abort on-going update of harvestable resources for selected harvesters") + def initiate_abort_update_harvestable_resources(self, request, queryset): + being_aborted = [] + for harvester in queryset: + try: + if harvester.update_availability(): + harvester.initiate_abort_update_harvestable_resources() + being_aborted.append(harvester) + else: + raise RuntimeError(f"Harvester {harvester!r} is not available") + except RuntimeError as exc: + self.message_user(request, str(exc), level=messages.ERROR) + if len(being_aborted) > 0: + message = ( + f"Aborting update of harvestable resources for {being_aborted}. " + f"This operation can take a while to complete. Check the harvesters' " + f"status for when it becomes `ready`" + ) + else: + message = _("No active refresh sessions have been found for the selected harvesters. Skipping...") + self.message_user(request, message) + + @admin.action(description="Perform harvesting on selected harvesters") + def initiate_perform_harvesting(self, request, queryset): + being_harvested = [] + for harvester in queryset: + try: + if harvester.update_availability(): + harvester.initiate_perform_harvesting() + being_harvested.append(harvester) + else: + raise RuntimeError(f"Harvester {harvester!r} is not available") + except RuntimeError as exc: + self.message_user(request, str(exc), level=messages.ERROR) + if len(being_harvested) > 0: + message = f"Performing harvesting asynchronously for {being_harvested}..." + else: + message = _("No ready harvesters have been selected, skipping...") + self.message_user(request, message) + + @admin.action(description="Abort on-going harvesting sessions for selected harvesters") + def initiate_abort_perform_harvesting(self, request, queryset): + being_aborted = [] + for harvester in queryset: + try: + if harvester.update_availability(): + harvester.initiate_abort_perform_harvesting() + being_aborted.append(harvester) + else: + raise RuntimeError(f"Harvester {harvester!r} is not available") + except RuntimeError as exc: + self.message_user(request, str(exc), level=messages.ERROR) + if len(being_aborted) > 0: + message = ( + f"Aborting current harvesting sessions for {being_aborted}. " + f"This operation can take a while to complete. Check the harvesters' " + f"status for when it becomes `ready`" + ) + else: + message = _("No active harvesting sessions have been found for the selected harvesters. Skipping...") + self.message_user(request, message) + + @admin.action(description="Reset harvester status") + def reset_harvester_status(self, request, queryset): + for harvester in queryset: + if harvester.status != models.Harvester.STATUS_READY: + harvester.status = models.Harvester.STATUS_READY + harvester.save() + self.message_user(request, _("Resetting status for harvester %(name)s...") % {"name": harvester.name}) + + @admin.display(description="Updating availability in") + def get_time_until_next_availability_update(self, harvester: models.Harvester): + return humanize.naturaltime(harvester.get_next_check_availability_dispatch_time()) + + @admin.display(description="Refreshing harvestable resources in") + def get_time_until_next_refresh(self, harvester: models.Harvester): + return humanize.naturaltime(harvester.get_next_refresh_session_dispatch_time()) + + @admin.display(description="Performing harvesting in") + def get_time_until_next_harvesting(self, harvester: models.Harvester): + return humanize.naturaltime(harvester.get_next_harvesting_session_dispatch_time()) + + @admin.display(description=_("Harvestable resources")) + def get_num_harvestable_resources(self, harvester: models.Harvester): + return harvester.num_harvestable_resources + + @admin.display(description="Worker-specific configuration") + def get_worker_specific_configuration(self, harvester: models.Harvester): + result = "
        " + try: + worker = harvester.get_harvester_worker() + worker_config = worker.get_current_config() + for key, value in worker_config.items(): + result += format_html("
      • {}: {}
        • ", key, value) + except Exception as e: + logger.exception(e) + result += "
      " + return mark_safe(result) + + @admin.display(description="Selected harvestable resources") + def show_link_to_selected_harvestable_resources(self, harvester: models.Harvester): + num_selected = models.HarvestableResource.objects.filter(harvester=harvester, should_be_harvested=True).count() + if num_selected > 0: + changelist_uri = reverse("admin:harvesting_harvestableresource_changelist") + result = mark_safe( + format_html( + f'({num_selected}) Go' + ) + ) + else: + result = None + return result + + @admin.display(description="Latest harvesting session") + def show_link_to_latest_harvesting_session(self, harvester: models.Harvester): + session = harvester.latest_harvesting_session + if session is not None: + changelist_uri = reverse( + "admin:harvesting_asynchronousharvestingsession_change", args=(harvester.latest_harvesting_session.id,) + ) + return mark_safe(format_html(f'Go')) + + @admin.display(description="Latest refresh session") + def show_link_to_latest_refresh_session(self, harvester: models.Harvester): + session = harvester.latest_refresh_session + if session is not None: + changelist_uri = reverse( + "admin:harvesting_asynchronousharvestingsession_change", args=(harvester.latest_refresh_session.id,) + ) + return mark_safe(format_html(f'Go')) + + +@admin.register(models.AsynchronousHarvestingSession) +class AsynchronousHarvestingSessionAdmin(admin.ModelAdmin): + list_display = ( + "id", + "session_type", + "status", + "started", + "updated", + "ended", + "harvester", + "total_records_to_process", + "records_done", + "get_progress_percentage", + ) + readonly_fields = ( + "id", + "session_type", + "status", + "started", + "updated", + "ended", + "harvester", + "total_records_to_process", + "records_done", + "get_progress_percentage", + "details", + ) + + def has_add_permission(self, request): + return False + + +@admin.register(models.HarvestableResource) +class HarvestableResourceAdmin(admin.ModelAdmin): + list_display = ( + "id", + "last_refreshed", + "last_harvesting_succeeded", + "last_harvested", + "unique_identifier", + "title", + "show_link_to_harvester", + "should_be_harvested", + "remote_resource_type", + ) + readonly_fields = ( + "unique_identifier", + "title", + "harvester", + "last_updated", + "last_refreshed", + "last_harvested", + "last_harvesting_message", + "last_harvesting_succeeded", + "remote_resource_type", + ) + list_filter = ( + "harvester", + "should_be_harvested", + "last_updated", + "remote_resource_type", + "last_harvesting_succeeded", + ) + search_fields = ("title",) + list_editable = ("should_be_harvested",) + + actions = [ + "toggle_should_be_harvested", + "initiate_harvest_selected_resources", + ] + + def delete_queryset(self, request, queryset): + """ + Re-implemented to assure individual instance's `delete()` method is called. + + `HarvestableResource.delete()` has some custom logic to check whether the + related GeoNode resource should also be deleted. Therefore we don't want Django + to potentially optimize this into performing the deletion on the DB, as that + would not run our custom logic. + + Further info: + + https://docs.djangoproject.com/en/3.2/topics/db/queries/#deleting-objects + + """ + + for harvestable_resource in queryset: + harvestable_resource.delete() + self.message_user(request, "Harvestable resources have been deleted") + + @admin.action(description="Toggle selected resources' `should_be_harvested` property") + def toggle_should_be_harvested(self, request, queryset): + for harvestable_resource in queryset: + harvestable_resource: models.HarvestableResource + harvestable_resource.should_be_harvested = not harvestable_resource.should_be_harvested + harvestable_resource.save() + + self.message_user(request, "Toggled harvestable resources' `should_be_harvested` attribute") + + @admin.action(description="Harvest selected resources") + def initiate_harvest_selected_resources(self, request, queryset): + selected_harvestable_resources = {} + for harvestable_resource in queryset: + harvester_resources = selected_harvestable_resources.setdefault(harvestable_resource.harvester, []) + harvester_resources.append(harvestable_resource.id) + for harvester, harvestable_resource_ids in selected_harvestable_resources.items(): + try: + if harvester.update_availability(): + harvester.initiate_perform_harvesting(harvestable_resource_ids) + self.message_user( + request, + f"Harvesting {len(harvestable_resource_ids)} resources from {harvester.name!r} harvester...", + ) + else: + raise RuntimeError(f"Harvester {harvester!r} is not available") + except RuntimeError as exc: + self.message_user(request, str(exc), level=messages.ERROR) + + @admin.display(description="harvester") + def show_link_to_harvester(self, harvestable_resource: models.HarvestableResource): + harvester = harvestable_resource.harvester + uri = reverse("admin:harvesting_harvester_change", args=(harvester.pk,)) + return mark_safe(format_html(f'{harvester.name}')) + + +def _worker_config_changed(form) -> bool: + field_name = "harvester_type_specific_configuration" + try: + original = json.loads(form.data[f"initial-{field_name}"]) + except KeyError: + result = True + else: + cleaned = form.cleaned_data.get(field_name) + result = original != cleaned + return result diff --git a/geonode/harvesting/api/__init__.py b/geonode/harvesting/api/__init__.py new file mode 100644 index 0000000..6b4db60 --- /dev/null +++ b/geonode/harvesting/api/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/harvesting/api/mixins.py b/geonode/harvesting/api/mixins.py new file mode 100644 index 0000000..647fbcc --- /dev/null +++ b/geonode/harvesting/api/mixins.py @@ -0,0 +1,42 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from rest_framework.request import Request +from rest_framework.response import Response + +logger = logging.getLogger(__name__) + + +class UpdateListModelMixin: + """Adds the `update_list` method to a viewset + + `update_list` is used by `api.routers.ListPatchRouter` in order to allow + performing PATCH requests against a viewset's `list` endpoint + """ + + def update_list(self, request: Request, *args, **kwargs): + serializer = self.get_serializer(data=request.data, partial=True, many=True) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) + return Response(serializer.data) + + def perform_update(self, serializer): + serializer.save() diff --git a/geonode/harvesting/api/routers.py b/geonode/harvesting/api/routers.py new file mode 100644 index 0000000..faf3bfb --- /dev/null +++ b/geonode/harvesting/api/routers.py @@ -0,0 +1,81 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from rest_framework.routers import ( + DynamicRoute, + Route, +) +from rest_framework_extensions.routers import ExtendedSimpleRouter + + +class ListPatchRouter(ExtendedSimpleRouter): + """A router that allows performing PATCH requests against the `list` endpoint + + This router is used in order to provide API endpoints that respond to PATCH + requsts against a viewset's `list` endpoint. It is useful for situations where it + is necessary to perform a batch update of multiple resources. For a use case, + consider the selection of which remote resources should be harvestable by the local + GeoNode: + + - User is shown a (paginated) list of resources that are available on the remote + service + - User must now choose which of these harvestable resources should be harvested + - Instead of forcing the user to make multiple PATCH requests to each harvestable + resource's detail page in order to set the resource's `should_be_harvested` + property, user can perform a PATCH request to + `harvesters/{harvester-id}/harvestable-resources` and set multiple resources' + `should_be_harvested` at the same time + + """ + + routes = [ + # List route. + Route( + url=r"^{prefix}{trailing_slash}$", + mapping={ + "get": "list", + "post": "create", + "patch": "update_list", + }, + name="{basename}-list", + detail=False, + initkwargs={"suffix": "List"}, + ), + # Dynamically generated list routes. Generated using + # @action(detail=False) decorator on methods of the viewset. + DynamicRoute( + url=r"^{prefix}/{url_path}{trailing_slash}$", name="{basename}-{url_name}", detail=False, initkwargs={} + ), + # Detail route. + Route( + url=r"^{prefix}/{lookup}{trailing_slash}$", + mapping={"get": "retrieve", "put": "update", "patch": "partial_update", "delete": "destroy"}, + name="{basename}-detail", + detail=True, + initkwargs={"suffix": "Instance"}, + ), + # Dynamically generated detail routes. Generated using + # @action(detail=True) decorator on methods of the viewset. + DynamicRoute( + url=r"^{prefix}/{lookup}/{url_path}{trailing_slash}$", + name="{basename}-{url_name}", + detail=True, + initkwargs={}, + ), + ] diff --git a/geonode/harvesting/api/serializers.py b/geonode/harvesting/api/serializers.py new file mode 100644 index 0000000..aa2579d --- /dev/null +++ b/geonode/harvesting/api/serializers.py @@ -0,0 +1,320 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import typing +import logging +import collections +import jsonschema.exceptions + +from django.contrib.auth import get_user_model +from dynamic_rest.serializers import DynamicModelSerializer +from django.core.exceptions import ValidationError as DjangoValidationError +from rest_framework import ( + exceptions, + serializers, +) +from rest_framework.fields import ( + get_error_detail, + set_value, + SkipField, +) +from rest_framework.reverse import reverse +from rest_framework.settings import api_settings + +from .. import ( + models, + tasks, +) + +logger = logging.getLogger(__name__) + + +class CurrentUserDefault: + requires_context = True + + def __call__(self, serializer_field): + return serializer_field.context["request"].user + + +class BriefHarvesterSerializer(DynamicModelSerializer): + class Meta: + model = models.Harvester + fields = ( + "id", + "name", + "status", + "remote_url", + "remote_available", + "scheduling_enabled", + "harvesting_session_update_frequency", + "refresh_harvestable_resources_update_frequency", + "check_availability_frequency", + "default_owner", + "links", + ) + + default_owner = serializers.PrimaryKeyRelatedField( + queryset=get_user_model().objects.all(), + default=CurrentUserDefault(), + ) + links = serializers.SerializerMethodField() + + def get_links(self, obj): + return { + "self": reverse( + "harvester-detail", + kwargs={ + "pk": obj.id, + }, + request=self.context["request"], + ), + "harvestable_resources": reverse( + "harvestable-resources-list", + kwargs={ + "harvester_id": obj.id, + }, + request=self.context["request"], + ), + } + + +class HarvesterSerializer(BriefHarvesterSerializer): + class Meta: + model = models.Harvester + fields = ( + "id", + "name", + "status", + "remote_url", + "remote_available", + "scheduling_enabled", + "check_availability_frequency", + "harvesting_session_update_frequency", + "refresh_harvestable_resources_update_frequency", + "default_owner", + "harvester_type", + "harvester_type_specific_configuration", + "last_checked_availability", + "last_checked_harvestable_resources", + "last_check_harvestable_resources_message", + "harvest_new_resources_by_default", + "delete_orphan_resources_automatically", + "last_updated", + "links", + ) + + def validate(self, data): + """Perform object-level validation + + In this method we implement validation of the following: + + - Check that the worker configuration is valid for the current worker type. + This is done by validating the config against the worker's extra config + jsonschema (if it exists) + - Check that the client does not try to change the object's and worker config + at the same time, as that could lead to invalid internal state (for example, + if the worker config changes we need to regenerate the list of harvestable + resources before allowing a new harvesting session to take place). + + """ + + worker_config_field = "harvester_type_specific_configuration" + worker_type_field = "harvester_type" + worker_type = data.get(worker_type_field, getattr(self.instance, worker_type_field, None)) + worker_config = data.get(worker_config_field, getattr(self.instance, worker_config_field, None)) + if worker_type is not None and worker_config is not None: + try: + models.validate_worker_configuration(worker_type, worker_config) + except jsonschema.exceptions.ValidationError: + raise serializers.ValidationError(f"Invalid {worker_config_field!r} configuration") + + if data.get("status") and data.get(worker_config_field): + raise serializers.ValidationError( + f"Cannot update a harvester's 'status' and {worker_config_field!r} at " f"the same time" + ) + return data + + def create(self, validated_data): + desired_status = validated_data.get("status", models.Harvester.STATUS_READY) + if desired_status != models.Harvester.STATUS_READY: + raise serializers.ValidationError( + f"Invalid initial value for status: {desired_status!r}. " + f"Either omit it or provide a " + f"value of {models.Harvester.STATUS_READY!r}" + ) + return super().create(validated_data) + + def update(self, instance: models.Harvester, validated_data): + """Update harvester and perform any required business logic as a side-effect. + + Updating the harvester's `status` attribute triggers additional work: + + - If `status` is set to + `models.Harvester.STATUS_UPDATING_HARVESTABLE_RESOURCES`, then we + proceed to refresh all harvestable resources related to the harvester + + - If `status` is set to `models.Harvester.STATUS_PERFORM_HARVESTING` + then we proceed by starting a new harvesting session + + - If `status` is set to `models.Harvester.STATUS_CHECKING_AVAILABILITY` + then we proceed to check the availability of the remote service + + Note that it is not possible for an API client to set a status of + `models.Harvester.STATUS_READY`. This status is set internally. + + Additional work can also be triggered when a change to the harvester worker + configuration is requested. In that case GeoNode must regenerate the list of + harvestable resources. + + Also note that all of these additional work items are carried out + asynchronously via celery tasks. + + """ + + desired_status = validated_data.get("status") + if not instance.status == models.Harvester.STATUS_READY: + raise serializers.ValidationError( + f"Harvester is currently busy. Please wait until current " + f"status becomes {models.Harvester.STATUS_READY!r}" + ) + elif desired_status == models.Harvester.STATUS_READY: + raise serializers.ValidationError( + f"Cannot set a status of {models.Harvester.STATUS_READY!r} explicitly. " + f"This status can only be set by the server, when appropriate." + ) + elif desired_status == models.Harvester.STATUS_UPDATING_HARVESTABLE_RESOURCES: + session = models.AsynchronousHarvestingSession.objects.create( + harvester=instance, + session_type=models.AsynchronousHarvestingSession.TYPE_DISCOVER_HARVESTABLE_RESOURCES, + ) + post_update_task = tasks.update_harvestable_resources.signature(args=(session.pk,)) + elif desired_status == models.Harvester.STATUS_PERFORMING_HARVESTING: + session = models.AsynchronousHarvestingSession.objects.create( + harvester=instance, session_type=models.AsynchronousHarvestingSession.TYPE_HARVESTING + ) + post_update_task = tasks.harvesting_dispatcher.signature(args=(session.pk,)) + elif desired_status == models.Harvester.STATUS_CHECKING_AVAILABILITY: + post_update_task = tasks.check_harvester_available.signature(args=(instance.id,)) + elif desired_status is None: + current_worker_config = instance.harvester_type_specific_configuration + desired_worker_config = validated_data.get("harvester_type_specific_configuration", current_worker_config) + if desired_worker_config != current_worker_config: + logger.debug(f"Regenerating harvestable resource list for harvester " f"{instance!r} asynchronously...") + models.HarvestableResource.objects.filter(harvester=instance).delete() + post_update_task = tasks.update_harvestable_resources.signature(args=(instance.id,)) + else: + post_update_task = None + else: + post_update_task = None + updated_instance = super().update(instance, validated_data) + if post_update_task is not None: + post_update_task.apply_async(args=(), expiration=30) + return updated_instance + + +class BriefAsynchronousHarvestingSessionSerializer(DynamicModelSerializer): + class Meta: + model = models.AsynchronousHarvestingSession + fields = ( + "id", + "started", + "updated", + "ended", + "total_records_to_process", + "records_done", + ) + + +class HarvestableResourceSerializer(DynamicModelSerializer): + class Meta: + model = models.HarvestableResource + fields = [ + "unique_identifier", + "title", + "should_be_harvested", + "last_updated", + "status", + "remote_resource_type", + ] + read_only_fields = [ + "title", + "last_updated", + "status", + "remote_resource_type", + ] + + def create(self, validated_data): + # NOTE: We are implementing `create()` rather than `update` intentionally, even if the + # user is not allowed to create new records (check the `views.py` module) - the rationale + # being that since we keep a harvestable_resource's `id` private it would be more involved + # to deal with its update than with its creation. We are providing a custom `UpdateListModelMixin` class + # that allows for bulk update of multiple instances simultaneously. This mixin class is instantiating + # this serializer class without providing an instance and then calling its `save()` method + harvestable_resource = models.HarvestableResource.objects.get( + harvester=self.context["harvester"], unique_identifier=validated_data["unique_identifier"] + ) + harvestable_resource.should_be_harvested = validated_data["should_be_harvested"] + harvestable_resource.save() + return harvestable_resource + + # TODO: need to check whether any worker-specific configuration has changed when the harvester is updated + # if so, then we need to discard existing harvestable_resources and check them again + + def to_internal_value(self, data): + """Verbatim copy of the original drf `to_internal_value()` method. + + This method replicates the implementation found on + `restframework.serializers.Serializer.to_internal_value` method because the + dynamic-rest package (which we are using, and which overrides the base + implementation) adds some custom stuff on top of it which depends on the input + data containing the instance's `id` property, which we are not requiring. + + A HarvestableResource's `id` is an implementation detail that is not exposed + publicly. We rely on the instance's `unique_identifier` instead. + + """ + + if not isinstance(data, typing.Mapping): + message = self.error_messages["invalid"].format(datatype=type(data).__name__) + raise exceptions.ValidationError({api_settings.NON_FIELD_ERRORS_KEY: [message]}, code="invalid") + + ret = collections.OrderedDict() + errors = collections.OrderedDict() + fields = self._writable_fields + + for field in fields: + validate_method = getattr(self, f"validate_{field.field_name}", None) + primitive_value = field.get_value(data) + try: + validated_value = field.run_validation(primitive_value) + if validate_method is not None: + validated_value = validate_method(validated_value) + except exceptions.ValidationError as exc: + errors[field.field_name] = exc.detail + except DjangoValidationError as exc: + errors[field.field_name] = get_error_detail(exc) + except SkipField: + pass + else: + set_value(ret, field.source_attrs, validated_value) + + if errors: + raise exceptions.ValidationError(errors) + + return ret diff --git a/geonode/harvesting/api/tests.py b/geonode/harvesting/api/tests.py new file mode 100644 index 0000000..f555e9e --- /dev/null +++ b/geonode/harvesting/api/tests.py @@ -0,0 +1,29 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import pytest +from rest_framework import status +from rest_framework.test import APIClient + + +@pytest.mark.django_db +def test_harvester_list(): + client = APIClient() + response = client.get("/api/v2/harvesters/") + assert response.status_code == status.HTTP_200_OK diff --git a/geonode/harvesting/api/urls.py b/geonode/harvesting/api/urls.py new file mode 100644 index 0000000..190bcbe --- /dev/null +++ b/geonode/harvesting/api/urls.py @@ -0,0 +1,36 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from . import ( + routers, + views, +) + +router = routers.ListPatchRouter() + +harvesters_node = router.register("harvesters", views.HarvesterViewSet) +harvesters_node.register( + "harvestable-resources", + views.HarvestableResourceViewSet, + basename="harvestable-resources", + parents_query_lookups=["harvester_id"], +) +router.register("harvesting-sessions", views.AsynchronousHarvestingSessionViewSet) + +urlpatterns = router.urls diff --git a/geonode/harvesting/api/views.py b/geonode/harvesting/api/views.py new file mode 100644 index 0000000..509d57f --- /dev/null +++ b/geonode/harvesting/api/views.py @@ -0,0 +1,93 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import rest_framework.permissions + +from dynamic_rest.viewsets import DynamicModelViewSet, WithDynamicViewSetMixin +from rest_framework.authentication import BasicAuthentication, SessionAuthentication +from rest_framework import mixins +from rest_framework import viewsets +from rest_framework_extensions.mixins import NestedViewSetMixin +from oauth2_provider.contrib.rest_framework import OAuth2Authentication +from geonode.base.api.pagination import GeoNodeApiPagination + +from .. import models +from . import serializers +from .mixins import UpdateListModelMixin + +import logging + +logger = logging.getLogger(__name__) + + +class IsAdminOrListOnly(rest_framework.permissions.BasePermission): + def has_permission(self, request, view): + if request.user.is_superuser: + result = True + elif view.action == "list": + result = True + else: + result = False + return result + + +class HarvesterViewSet(NestedViewSetMixin, DynamicModelViewSet): + authentication_classes = [ + BasicAuthentication, + SessionAuthentication, + OAuth2Authentication, + ] + permission_classes = [ + IsAdminOrListOnly, + ] + queryset = models.Harvester.objects.all() + serializer_class = serializers.HarvesterSerializer + pagination_class = GeoNodeApiPagination + + def get_serializer_class(self): + if self.action == "list": + result = serializers.BriefHarvesterSerializer + else: + result = serializers.HarvesterSerializer + return result + + +class HarvestableResourceViewSet( + UpdateListModelMixin, + NestedViewSetMixin, + WithDynamicViewSetMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): + queryset = models.HarvestableResource.objects.all() + pagination_class = GeoNodeApiPagination + serializer_class = serializers.HarvestableResourceSerializer + + def get_serializer_context(self): + context = super().get_serializer_context() + logger.debug(f"serializer_context: {context}") + logger.debug(f"self.request: {self.request}") + harvester = models.Harvester.objects.get(pk=self.kwargs["harvester_id"]) + context["harvester"] = harvester + return context + + +class AsynchronousHarvestingSessionViewSet(WithDynamicViewSetMixin, viewsets.ReadOnlyModelViewSet): + queryset = models.AsynchronousHarvestingSession.objects.all() + serializer_class = serializers.BriefAsynchronousHarvestingSessionSerializer + pagination_class = GeoNodeApiPagination diff --git a/geonode/harvesting/apps.py b/geonode/harvesting/apps.py new file mode 100644 index 0000000..97b8861 --- /dev/null +++ b/geonode/harvesting/apps.py @@ -0,0 +1,38 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.apps import AppConfig +from django.conf import settings +from django.urls import include, re_path + +from . import config + + +class HarvestingAppConfig(AppConfig): + name = "geonode.harvesting" + + def ready(self): + from geonode.urls import urlpatterns + from . import signals # noqa + + urlpatterns += [re_path(r"^api/v2/", include("geonode.harvesting.api.urls"))] + settings.CELERY_BEAT_SCHEDULE["harvesting-scheduler"] = { + "task": "geonode.harvesting.tasks.harvesting_scheduler", + "schedule": config.get_setting("HARVESTER_SCHEDULER_FREQUENCY_MINUTES") * 60, + } diff --git a/geonode/harvesting/config.py b/geonode/harvesting/config.py new file mode 100644 index 0000000..2b211c8 --- /dev/null +++ b/geonode/harvesting/config.py @@ -0,0 +1,54 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""Configuration for the harvesting app + +This module provides sensible default values for the harvesting app. + +""" + +import typing + +from django.conf import settings + + +_DEFAULT_HARVESTERS: typing.Final = [ + "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker", + "geonode.harvesting.harvesters.wms.OgcWmsHarvester", + "geonode.harvesting.harvesters.arcgis.ArcgisHarvesterWorker", +] + + +def _get_harvester_class_paths(custom_class_paths: typing.List[str]) -> typing.List[str]: + result = _DEFAULT_HARVESTERS[:] + for i in custom_class_paths: + if i not in result: + result.append(i) + return result + + +def get_setting(setting_key: str) -> typing.Any: + result = { + "HARVESTER_CLASSES": _get_harvester_class_paths(getattr(settings, "HARVESTER_CLASSES", [])), + "HARVESTED_RESOURCE_FILE_MAX_MEMORY_SIZE": getattr( + settings, "HARVESTED_RESOURCE_MAX_MEMORY_SIZE", settings.FILE_UPLOAD_MAX_MEMORY_SIZE + ), + "HARVESTER_SCHEDULER_FREQUENCY_MINUTES": getattr(settings, "HARVESTER_SCHEDULER_FREQUENCY_MINUTES", 0.5), + }.get(setting_key, getattr(settings, setting_key, None)) + return result diff --git a/geonode/harvesting/forms.py b/geonode/harvesting/forms.py new file mode 100644 index 0000000..7834094 --- /dev/null +++ b/geonode/harvesting/forms.py @@ -0,0 +1,37 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django import forms + +from . import models + +logger = logging.getLogger(__name__) + + +class HarvesterForm(forms.ModelForm): + def clean(self): + super().clean() + if self.instance.status != models.Harvester.STATUS_READY: + raise forms.ValidationError( + f"Harvester is currently busy. Please wait until current " + f"status becomes {models.Harvester.STATUS_READY!r} before retrying" + ) + return self.cleaned_data diff --git a/geonode/harvesting/harvesters/__init__.py b/geonode/harvesting/harvesters/__init__.py new file mode 100644 index 0000000..6b4db60 --- /dev/null +++ b/geonode/harvesting/harvesters/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/harvesting/harvesters/arcgis.py b/geonode/harvesting/harvesters/arcgis.py new file mode 100644 index 0000000..46054a6 --- /dev/null +++ b/geonode/harvesting/harvesters/arcgis.py @@ -0,0 +1,565 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""Harvesters for ArcGIS based remote servers.""" +import re +import abc +import enum +import json +import logging +import typing +import uuid +from urllib.error import ( + HTTPError, + URLError, +) +import urllib.parse + +import arcrest +import requests +from django.contrib.gis import geos +from django.template.defaultfilters import slugify + +from geonode.layers.enumerations import GXP_PTYPES +from geonode.layers.models import Dataset + +from .. import ( + models, + resourcedescriptor, +) + +from . import base + +logger = logging.getLogger(__name__) + + +class ArcgisRestApiLayerType(enum.Enum): + GROUP_LAYER = "Group Layer" + FEATURE_LAYER = "Feature Layer" + + +class ArcgisServiceType(enum.Enum): + MAP_SERVICE = "MapServer" + FEATURE_SERVICE = "FeatureServer" + GEOCODE_SERVICE = "GeocodeServer" + GEOPROCESSING_SERVICE = "GPServer" + GEOMETRY_SERVICE = "GeometryServer" + IMAGE_SERVICE = "ImageServer" + NETWORK_SERVICE = "NAServer" + GEODATA_SERVICE = "GeoDataServer" + GLOBE_SERVICE = "GlobeServer" + MOBILE_SERVICE = "MobileServer" + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + +def parse_remote_url(url: str) -> typing.Tuple[str, typing.Optional[str], typing.Optional[str]]: + """Parse the input url into the ArcGIS REST catalog URL and any service name.""" + url_fragments = url.partition("/rest/services") + catalog_url = "".join(url_fragments[:2]) + service_type = None + possible_service_name = None + service_type_regex = re.match(r".*\/(.*Server).*", "".join(url_fragments[-1:])) + if service_type_regex: + for service_type_value in service_type_regex.groups(): + if ArcgisServiceType.has_value(service_type_value): + service_type = service_type_value + possible_service_name = "".join(url_fragments[-1:]).strip("/").partition(service_type)[0].rstrip("/") + other = None + break + else: + possible_service_name, other = url_fragments[-1].strip("/").partition("/")[::2] + if possible_service_name is not None and possible_service_name != "": + service_name = possible_service_name + if not service_type and other: + service_type = other.partition("/")[0] + else: + service_name = None + + return catalog_url, service_name, service_type + + +class ArcgisServiceResourceExtractor(abc.ABC): + """Abstract base class with the methods that must be reimplemented + in order to add support for additional ArcGIS REST services""" + + resource_name_filter: typing.Optional[str] + service: typing.Type + + def __init__(self, service, resource_name_filter: typing.Optional[str] = None): + self.service = service + self.resource_name_filter = resource_name_filter + + @abc.abstractmethod + def get_num_resources(self) -> int: + """Return the number of resources that can be extracted from the service.""" + + @abc.abstractmethod + def list_resources(self) -> typing.List[base.BriefRemoteResource]: + """Return a list of BriefRemoteResource with the resources exposed by the service""" + + @abc.abstractmethod + def get_resource(self, harvestable_resource: models.HarvestableResource) -> base.HarvestedResourceInfo: + """Parse the remote resource into a HarvestedResourceInfo""" + + def _is_relevant_layer(self, layer_name: str) -> bool: + result = False + if self.resource_name_filter is not None: + if self.resource_name_filter.lower() in layer_name.lower(): + result = True + else: + result = True + return result + + +class ArcgisMapServiceResourceExtractor(ArcgisServiceResourceExtractor): + service: arcrest.MapService + http_session: requests.Session + _cached_resources: typing.Optional[typing.List[base.BriefRemoteResource]] + + def __init__(self, service: arcrest.MapService): + super().__init__(service) + self.http_session = requests.Session() + self._cached_resources = None + + def get_num_resources(self) -> int: + if self._cached_resources is None: + self._cached_resources = self._extract_resources() + return len(self._cached_resources) + + def list_resources( + self, + ) -> typing.List[base.BriefRemoteResource]: + if self._cached_resources is None: + self._cached_resources = self._extract_resources() + return self._cached_resources + + def get_resource(self, harvestable_resource: models.HarvestableResource): + response = self.http_session.get(harvestable_resource.unique_identifier, params={"f": "json"}) + result = None + if response.status_code == requests.codes.ok: + try: + response_payload = response.json() + except json.JSONDecodeError: + logger.exception("Could not decode response payload as valid JSON") + else: + resource_descriptor = self._get_resource_descriptor(response_payload, harvestable_resource) + result = base.HarvestedResourceInfo( + resource_descriptor=resource_descriptor, additional_information=None + ) + else: + logger.error( + f"Could not retrieve remote resource with unique " + f"identifier {harvestable_resource.unique_identifier!r}" + ) + return result + + def _extract_resources(self) -> typing.List[base.BriefRemoteResource]: + result = [] + try: + for arc_layer in self.service.layers: + if ( + self._is_relevant_layer(arc_layer.name) + and arc_layer.type != ArcgisRestApiLayerType.GROUP_LAYER.value + ): + result.append(self._parse_brief_layer(arc_layer)) + result.extend(self._list_sub_layers(arc_layer)) + except HTTPError: + logger.exception(msg="Could not list resources") + return result + + def _list_sub_layers(self, arc_layer: arcrest.MapLayer) -> typing.List[base.BriefRemoteResource]: + result = [] + for sub_layer in arc_layer.subLayers: + if self._is_relevant_layer(arc_layer.name) and arc_layer.type != ArcgisRestApiLayerType.GROUP_LAYER.value: + result.append(self._parse_brief_layer(sub_layer)) + result.extend(self._list_sub_layers(sub_layer)) + return result + + def _get_resource_descriptor( + self, layer_representation: typing.Dict, harvestable_resource: models.HarvestableResource + ) -> resourcedescriptor.RecordDescription: + if harvestable_resource.geonode_resource is None: + resource_uuid = uuid.uuid4() + else: + resource_uuid = uuid.UUID(harvestable_resource.geonode_resource.uuid) + _, service_name, service_type = parse_remote_url(harvestable_resource.unique_identifier) + epsg_code, spatial_extent = _parse_spatial_extent(layer_representation["extent"]) + ows_url = harvestable_resource.harvester.remote_url + store = slugify(ows_url) + name = layer_representation.get("id", layer_representation.get("name", "Undefined")) + title = layer_representation.get("name", layer_representation.get("title", "Undefined")) + workspace = "remoteWorkspace" + alternate = f"{workspace}:{name}" + return resourcedescriptor.RecordDescription( + uuid=resource_uuid, + identification=resourcedescriptor.RecordIdentification( + name=name, + title=title, + abstract=layer_representation.get("description", ""), + other_constraints=layer_representation.get("copyrightTest", ""), + spatial_extent=spatial_extent, + other_keywords=[ + "ESRI", + f"ArcGIS REST {self.service.__service_type__}", + ], + ), + distribution=resourcedescriptor.RecordDistribution( + link_url=harvestable_resource.unique_identifier, + thumbnail_url=None, + ), + reference_systems=[epsg_code], + additional_parameters={ + "store": store, + "workspace": workspace, + "alternate": alternate, + "ows_url": ows_url, + "ptype": GXP_PTYPES["REST_MAP"], + }, + ) + + def _parse_brief_layer(self, arc_layer: arcrest.MapLayer) -> base.BriefRemoteResource: + base_url = urllib.parse.urlparse(self.service.url) + layer_path = "/".join((base_url.path.rstrip("/"), str(arc_layer.id))) + layer_url = urllib.parse.urlunparse((base_url.scheme, base_url.netloc, layer_path, "", "", "")) + return base.BriefRemoteResource( + unique_identifier=layer_url, + title=arc_layer.name, + resource_type=arc_layer.type, + ) + + +class ArcgisImageServiceResourceExtractor(ArcgisServiceResourceExtractor): + service: arcrest.ImageService + http_session: requests.Session + + def __init__(self, service: arcrest.ImageService): + super().__init__(service) + self.http_session = requests.Session() + + def get_num_resources(self) -> int: + return len(self.list_resources()) + + def list_resources(self) -> typing.List[base.BriefRemoteResource]: + name = self._get_resource_name() + if self._is_relevant_layer(name): + unique_id = self.service.url.rpartition("?")[0].rstrip("/") + result = [ + base.BriefRemoteResource( + unique_identifier=unique_id, + title=name, + resource_type="raster", + ) + ] + else: + result = [] + return result + + def get_resource(self, harvestable_resource: models.HarvestableResource) -> base.HarvestedResourceInfo: + response = self.http_session.get(harvestable_resource.unique_identifier, params={"f": "json"}) + result = None + if response.status_code == requests.codes.ok: + try: + response_payload = response.json() + except json.JSONDecodeError: + logger.exception("Could not decode response payload as valid JSON") + else: + resource_descriptor = self._get_resource_descriptor(response_payload, harvestable_resource) + result = base.HarvestedResourceInfo( + resource_descriptor=resource_descriptor, additional_information=None + ) + else: + logger.error( + f"Could not retrieve remote resource with unique " + f"identifier {harvestable_resource.unique_identifier!r}" + ) + return result + + def _get_resource_name(self): + return self.service.url.rpartition("/rest/services/")[-1].partition("/ImageServer")[0] + + def _get_resource_descriptor( + self, layer_representation: typing.Dict, harvestable_resource: models.HarvestableResource + ) -> resourcedescriptor.RecordDescription: + if harvestable_resource.geonode_resource is None: + resource_uuid = uuid.uuid4() + else: + resource_uuid = uuid.UUID(harvestable_resource.geonode_resource.uuid) + _, service_name, service_type = parse_remote_url(harvestable_resource.unique_identifier) + epsg_code, spatial_extent = _parse_spatial_extent(layer_representation["extent"]) + ows_url = harvestable_resource.harvester.remote_url + store = slugify(ows_url) + name = layer_representation.get("id", layer_representation.get("name", "Undefined")) + title = layer_representation.get("name", layer_representation.get("title", "Undefined")) + workspace = "remoteWorkspace" + alternate = f"{workspace}:{name}" + return resourcedescriptor.RecordDescription( + uuid=resource_uuid, + identification=resourcedescriptor.RecordIdentification( + name=name, + title=title, + abstract=layer_representation.get("description", ""), + other_constraints=layer_representation.get("copyrightTest", ""), + spatial_extent=spatial_extent, + other_keywords=[ + "ESRI", + f"ArcGIS REST {self.service.__service_type__}", + ], + ), + distribution=resourcedescriptor.RecordDistribution( + link_url=harvestable_resource.unique_identifier, + thumbnail_url=None, + ), + reference_systems=[epsg_code], + additional_parameters={ + "store": store, + "workspace": workspace, + "alternate": alternate, + "ows_url": ows_url, + "ptype": GXP_PTYPES["REST_IMG"], + }, + ) + + +def get_resource_extractor(resource_unique_identifier: str) -> typing.Optional[ArcgisServiceResourceExtractor]: + """A factory for instantiating the correct extractor for the resource""" + service_type_name = parse_remote_url(resource_unique_identifier)[-1] + service_type = ArcgisServiceType(service_type_name) + if service_type == ArcgisServiceType.MAP_SERVICE: + service = arcrest.MapService(resource_unique_identifier) + result = ArcgisMapServiceResourceExtractor(service) + elif service_type == ArcgisServiceType.IMAGE_SERVICE: + service = arcrest.ImageService(resource_unique_identifier) + result = ArcgisImageServiceResourceExtractor(service) + else: + logger.error(f"Unsupported ArcGIS REST service {service_type!r}") + result = None + return result + + +class ArcgisHarvesterWorker(base.BaseHarvesterWorker): + harvest_map_services: bool + harvest_image_services: bool + resource_name_filter: typing.Optional[str] + service_names_filter: typing.Optional[typing.List[str]] + + http_session: requests.Session + _arc_catalog: typing.Optional[arcrest.Catalog] + _relevant_service_extractors: typing.Optional[ + typing.List[typing.Union[ArcgisMapServiceResourceExtractor, ArcgisImageServiceResourceExtractor]] + ] + _supported_service_types = { + ArcgisServiceType.MAP_SERVICE: ArcgisMapServiceResourceExtractor, + ArcgisServiceType.IMAGE_SERVICE: ArcgisImageServiceResourceExtractor, + } + + def __init__( + self, + remote_url: str, + harvester_id: int, + harvest_map_services: bool = True, + harvest_image_services: bool = True, + resource_name_filter: typing.Optional[str] = True, + service_names_filter: typing.Optional[typing.List[str]] = None, + ) -> None: + catalog_url, service_name, service_type_name = parse_remote_url(remote_url) + if service_name is not None: + names_filter = [service_name] + (service_names_filter or []) + service_type = ArcgisServiceType(service_type_name) + harvest_maps = (service_type == ArcgisServiceType.MAP_SERVICE) or harvest_map_services + harvest_images = (service_type == ArcgisServiceType.IMAGE_SERVICE) or harvest_image_services + else: + names_filter = service_names_filter or [] + harvest_maps = harvest_map_services + harvest_images = harvest_image_services + super().__init__(catalog_url, harvester_id) + self.http_session = requests.Session() + self.harvest_map_services = harvest_maps + self.harvest_image_services = harvest_images + self.resource_name_filter = resource_name_filter + self.service_names_filter = names_filter + self._arc_catalog = None + self._relevant_service_extractors = None + + @property + def allows_copying_resources(self) -> bool: + return False + + @property + def arc_catalog(self): + if self._arc_catalog is None: + try: + self._arc_catalog = arcrest.Catalog(self.remote_url) + except (json.JSONDecodeError, URLError, HTTPError): + logger.exception(f"Could not connect to ArcGIS REST server at {self.remote_url!r}") + return self._arc_catalog + + @classmethod + def from_django_record(cls, harvester: "Harvester"): # noqa + return cls( + remote_url=harvester.remote_url, + harvester_id=harvester.pk, + harvest_map_services=harvester.harvester_type_specific_configuration.get("harvest_map_services", True), + harvest_image_services=harvester.harvester_type_specific_configuration.get("harvest_image_services", True), + resource_name_filter=harvester.harvester_type_specific_configuration.get("resource_name_filter"), + service_names_filter=harvester.harvester_type_specific_configuration.get("service_names_filter"), + ) + + @classmethod + def get_extra_config_schema(cls) -> typing.Optional[typing.Dict]: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": ("https://geonode.org/harvesting/geonode-arcgis-rest-harvester.schema.json"), + "title": "ArcGIS REST harvester config", + "description": ( + "A jsonschema for validating configuration option for GeoNode's " + "remote ArcGIS REST services harvester" + ), + "type": "object", + "properties": { + "harvest_map_services": {"type": "boolean", "default": True}, + "harvest_image_services": {"type": "boolean", "default": True}, + "resource_name_filter": { + "type": "string", + }, + "service_names_filter": { + "type": "array", + "items": { + "type": "string", + }, + }, + }, + "additionalProperties": False, + } + + def get_num_available_resources(self) -> int: + result = 0 + for service_extractor in self._get_relevant_services(): + result += service_extractor.get_num_resources() + return result + + def list_resources(self, offset: typing.Optional[int] = 0) -> typing.List[base.BriefRemoteResource]: + result = [] + # NOTE: Since ArcGIS REST services work in a nested fashion we are + # not able to paginate the underlying results. As such, we resort to + # processing all resources sequentially. This means we only care about + # `offset=0` and explicitly return an empty list when the supplied + # offset is different. + if offset == 0: + for service_extractor in self._get_relevant_services(): + result.extend(service_extractor.list_resources()) + return result + + def check_availability(self, timeout_seconds: typing.Optional[int] = 5) -> bool: + return self.arc_catalog is not None + + def get_geonode_resource_type(self, remote_resource_type: str) -> typing.Type: + return Dataset + + def get_geonode_resource_defaults( + self, + harvested_info: base.HarvestedResourceInfo, + harvestable_resource: models.HarvestableResource, + ) -> typing.Dict: + defaults = super().get_geonode_resource_defaults(harvested_info, harvestable_resource) + defaults["name"] = harvested_info.resource_descriptor.identification.name + defaults.update(harvested_info.resource_descriptor.additional_parameters) + return defaults + + def get_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> typing.Optional[base.HarvestedResourceInfo]: + extractor = get_resource_extractor(harvestable_resource.unique_identifier) + extractor.resource_name_filter = self.resource_name_filter + return extractor.get_resource(harvestable_resource) + + def _get_extractor_class(self, service_type: ArcgisServiceType) -> typing.Optional[typing.Type]: + if service_type == ArcgisServiceType.MAP_SERVICE and self.harvest_map_services: + result = ArcgisMapServiceResourceExtractor + elif service_type == ArcgisServiceType.IMAGE_SERVICE and self.harvest_image_services: + result = ArcgisImageServiceResourceExtractor + else: + result = None + return result + + def _get_service_extractors(self, service) -> typing.List: + # This method is fugly. Unfortunately, when multiple services share the + # same name, arcrest just instantiates an `AmbiguousService` instance and + # shoves the concrete services as attributes of this instance. + # To make matters more unpleasant, the arcrest `AmbiguousService` class is + # defined inside the `__getitem__` method of another class, so it cannot be + # imported outside of it. Thus we resort to checking if there is a + # `__service_type__` attribute on the service in order to deduct whether this is a + # legit service or an ambiguous one and then deal with it + result = [] + if not hasattr(service, "__service_type__"): + # this is an arcrest AmbiguousService instance + for sub_service_type in service.__dict__.keys(): + try: + type_ = ArcgisServiceType(sub_service_type) + except ValueError: + logger.debug(f"Unrecognized service type: {sub_service_type!r}") + continue + else: + extractor_class = self._get_extractor_class(type_) + if extractor_class is not None: + sub_service = getattr(service, sub_service_type) + extractor = extractor_class(sub_service) + extractor.resource_name_filter = self.resource_name_filter + result.append(extractor) + else: + try: + type_ = ArcgisServiceType(service.__service_type__) + except ValueError: + logger.debug(f"Unrecognized service type: {service.__service_type__!r}") + else: + extractor_class = self._get_extractor_class(type_) + if extractor_class is not None: + extractor = extractor_class(service) + extractor.resource_name_filter = self.resource_name_filter + result.append(extractor) + return result + + def _get_relevant_services( + self, + ) -> typing.List[typing.Union[ArcgisMapServiceResourceExtractor, ArcgisImageServiceResourceExtractor]]: + if self._relevant_service_extractors is None: + result = [] + relevant_service_names = self.service_names_filter or self.arc_catalog.servicenames + for service_name in relevant_service_names: + service = None + for _folder in service_name.split("/"): + if not service: + service = self.arc_catalog[_folder] + else: + service = service[_folder] + extractors = self._get_service_extractors(service) + result.extend(extractors) + self._relevant_service_extractors = result + return self._relevant_service_extractors + + +def _parse_spatial_extent(raw_extent: typing.Dict) -> typing.Tuple[str, geos.Polygon]: + spatial_reference = raw_extent.get("spatialReference", {}) + epsg_code = f"EPSG:{spatial_reference.get('latestWkid', spatial_reference.get('wkid'))}" + extent = geos.Polygon.from_bbox((raw_extent["xmin"], raw_extent["ymin"], raw_extent["xmax"], raw_extent["ymax"])) + return epsg_code, extent diff --git a/geonode/harvesting/harvesters/base.py b/geonode/harvesting/harvesters/base.py new file mode 100644 index 0000000..8c273f5 --- /dev/null +++ b/geonode/harvesting/harvesters/base.py @@ -0,0 +1,374 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import abc +import dataclasses +import html +import io +import logging +import typing +from pathlib import Path + +import geonode.upload.files +import requests +from django.core.files import uploadedfile + +from geonode.base import enumerations +from geonode.base.models import ResourceBase +from geonode.resource.manager import resource_manager +from geonode.storage.manager import storage_manager + +from .. import ( + config, + resourcedescriptor, +) + +logger = logging.getLogger(__name__) + + +class HarvestingException(Exception): + pass + + +@dataclasses.dataclass() +class BriefRemoteResource: + unique_identifier: str + title: str + resource_type: str + abstract: typing.Optional[str] = "" + should_be_harvested: bool = False + + +@dataclasses.dataclass() +class HarvestedResourceInfo: + resource_descriptor: resourcedescriptor.RecordDescription + additional_information: typing.Optional[typing.Any] + copied_resources: typing.Optional[typing.List] = dataclasses.field(default_factory=list) + + +class BaseHarvesterWorker(abc.ABC): + """Base class for harvesters. + + This provides two relevant things: + + - An interface that all custom GeoNode harvesting classes must implement; + - Default implementation for common functionality. + + """ + + remote_url: str + harvester_id: int + + def __init__(self, remote_url: str, harvester_id: int): + self.remote_url = remote_url + self.harvester_id = harvester_id + + @property + @abc.abstractmethod + def allows_copying_resources(self) -> bool: + """Whether copying remote resources is implemented by this worker""" + + @classmethod + @abc.abstractmethod + def from_django_record(cls, harvester: "Harvester"): # noqa + """Return a new instance of the worker from the django harvester""" + + @abc.abstractmethod + def get_num_available_resources(self) -> int: + """Return the number of available resources on the remote service. + + If there is a problem retrieving the number of available resource, this + method shall raise `HarvestingException`. + + """ + + @abc.abstractmethod + def list_resources(self, offset: typing.Optional[int] = 0) -> typing.List[BriefRemoteResource]: + """Return a list of resources from the remote service. + + If there is a problem listing resource, this method shall + raise `HarvestingException`. + + """ + + @abc.abstractmethod + def check_availability(self, timeout_seconds: typing.Optional[int] = 5) -> bool: + """Check whether the remote service is online""" + + @abc.abstractmethod + def get_geonode_resource_type(self, remote_resource_type: str) -> ResourceBase: + """ + Return the GeoNode type that should be created from the remote resource type + """ + + @abc.abstractmethod + def get_resource( + self, + harvestable_resource: "HarvestableResource", # noqa + ) -> typing.Optional[HarvestedResourceInfo]: + """Harvest a single resource from the remote service. + + The return value is an instance of `HarvestedResourceInfo`. It stores + an instance of `RecordDescription` and additionally whatever type is + required by child classes to be able to customize resource creation/update on + the local GeoNode. Note that the default implementation of `update_geonode_resource()` + only needs the `RecordDescription`. The possibility to return additional information + exists solely for extensibility purposes and can be left as None in the simple cases. + + """ + + @classmethod + def get_extra_config_schema(cls) -> typing.Optional[typing.Dict]: + """Return a jsonschema schema to be used to validate models.Harvester objects""" + return None + + def get_current_config(self) -> typing.Dict: + """Return a dict with the current configuration.""" + schema = self.get_extra_config_schema() + result = {} + for property_name in schema["properties"]: + result[property_name] = getattr(self, property_name, None) + return result + + def finalize_resource_update( + self, + geonode_resource: ResourceBase, + harvested_info: HarvestedResourceInfo, + harvestable_resource: "HarvestableResource", # noqa + ) -> ResourceBase: + """Perform additional actions just after having created/updated a local GeoNode resource. + + This method can be used to further manipulate the relevant GeoNode Resource that is being + created/updated in the context of a harvesting operation. It is typically called from within + `base.BaseHarvesterWorker.update_geonode_resource` as the last step, after having already + acted upon the GeoNode resource. + The default implementation does nothing. + + """ + + return geonode_resource + + def finalize_harvestable_resource_deletion(self, harvestable_resource: "HarvestableResource") -> bool: # noqa + """Perform additional actions just before deleting a harvestable resource. + + This method is typically called from within `models.HarvestableResource.delete()`, just before + deleting the actual harvestable resource. It can be useful for child classes that customize + resource creation in order to also customize the deletion of harvestable resources. + The default implementation does nothing. + + """ + + return True + + def should_copy_resource( + self, + harvestable_resource: "HarvestableResource", # noqa + ) -> bool: + """Return True if the worker is able to copy the remote resource. + + The base implementation just returns False. Subclasses must re-implement this method + if they support copying remote resources onto the local GeoNode. + + """ + + return False + + def copy_resource( + self, + harvestable_resource: "HarvestableResource", # noqa + harvested_resource_info: HarvestedResourceInfo, + ) -> typing.Optional[Path]: + """Copy a remote resource's data to the local GeoNode. + + The base implementation provides a generic copy using GeoNode's `storage_manager`. + Subclasses may need to re-implement this method if they require specialized behavior. + + """ + + url = harvested_resource_info.resource_descriptor.distribution.download_url + result = None + if url is not None: + target_name = _get_file_name(harvested_resource_info) + final_name = "/".join((str(harvested_resource_info.resource_descriptor.uuid), target_name)) + try: + result = download_resource_file(url, final_name) + except requests.exceptions.HTTPError: + logger.exception(f"Could not download resource file from {url!r}") + else: + logger.warning("harvested resource info does not provide a URL for retrieving the " "resource, skipping...") + return result + + def get_geonode_resource_defaults( + self, + harvested_info: HarvestedResourceInfo, + harvestable_resource: "HarvestableResource", # noqa + ) -> typing.Dict: + """ + Extract default values to be used by resource manager when updating a resource + """ + + defaults = { + "owner": harvestable_resource.harvester.default_owner, + "uuid": str(harvested_info.resource_descriptor.uuid), + "abstract": harvested_info.resource_descriptor.identification.abstract, + "bbox_polygon": harvested_info.resource_descriptor.identification.spatial_extent, + "srid": harvested_info.resource_descriptor.reference_systems[0], + "constraints_other": harvested_info.resource_descriptor.identification.other_constraints, + "created": harvested_info.resource_descriptor.date_stamp, + "data_quality_statement": harvested_info.resource_descriptor.data_quality, + "date": harvested_info.resource_descriptor.identification.date, + "date_type": harvested_info.resource_descriptor.identification.date_type, + "language": harvested_info.resource_descriptor.language, + "purpose": harvested_info.resource_descriptor.identification.purpose, + "supplemental_information": (harvested_info.resource_descriptor.identification.supplemental_information), + "title": harvested_info.resource_descriptor.identification.title, + "files": [str(path) for path in harvested_info.copied_resources], + "thumbnail_url": harvested_info.resource_descriptor.distribution.thumbnail_url, + } + if harvested_info.resource_descriptor.identification.lonlat_extent: + defaults["ll_bbox_polygon"] = harvested_info.resource_descriptor.identification.lonlat_extent + if self.should_copy_resource(harvestable_resource): + defaults["sourcetype"] = enumerations.SOURCE_TYPE_COPYREMOTE + else: + defaults["subtype"] = "remote" + defaults["sourcetype"] = enumerations.SOURCE_TYPE_REMOTE + return {key: value for key, value in defaults.items() if value is not None} + + def update_geonode_resource( + self, + harvested_info: HarvestedResourceInfo, + harvestable_resource: "HarvestableResource", # noqa + ): + """Create or update a local GeoNode resource with the input harvested information. + + If the underlying harvestable resource already exists as a local GeoNode resource, then + it is updated. Otherwise it is created locally. + + If something goes wrong with the update of the geonode resource this method should raise + a `RuntimeError`. This will be caught by the harvesting task and handled appropriately. + + """ + + defaults = self.get_geonode_resource_defaults(harvested_info, harvestable_resource) + geonode_resource = harvestable_resource.geonode_resource + if geonode_resource is None: + geonode_resource_type = self.get_geonode_resource_type(harvestable_resource.remote_resource_type) + geonode_resource = self._create_new_geonode_resource(geonode_resource_type, defaults) + elif not geonode_resource.uuid == str(harvested_info.resource_descriptor.uuid): + raise RuntimeError( + f"Resource {geonode_resource!r} already exists locally but its " + f"UUID ({geonode_resource.uuid}) does not match the one found on " + f"the remote resource {harvested_info.resource_descriptor.uuid!r}" + ) + else: + geonode_resource = self._update_existing_geonode_resource(geonode_resource, defaults) + keywords = _consolidate_resource_keywords( + harvested_info.resource_descriptor, geonode_resource, self.harvester_id + ) + regions = harvested_info.resource_descriptor.identification.place_keywords + # Make sure you set the "harvestable_resource.geonode_resource" before calling the "resource_manager" + harvestable_resource.geonode_resource = geonode_resource + harvestable_resource.save() + geonode_resource = resource_manager.update( + str(harvested_info.resource_descriptor.uuid), regions=regions, keywords=keywords + ) + self.finalize_resource_update( + geonode_resource, + harvested_info, + harvestable_resource, + ) + + def _create_new_geonode_resource(self, geonode_resource_type, defaults: typing.Dict): + logger.debug(f"Creating a new GeoNode resource for resource with uuid: {defaults['uuid']!r}...") + resource_defaults = defaults.copy() + + logger.debug("calling resource_manager.create...") + geonode_resource = resource_manager.create(resource_defaults["uuid"], geonode_resource_type, defaults) + return geonode_resource + + def _update_existing_geonode_resource(self, geonode_resource: ResourceBase, defaults: typing.Dict): + resource_defaults = defaults.copy() + if len(resource_defaults.get("files", [])) > 0: + result = resource_manager.replace(geonode_resource, vals=resource_defaults) + else: + result = resource_manager.update(geonode_resource.uuid, vals=resource_defaults) + return result + + +def download_resource_file(url: str, target_name: str) -> Path: + """Download a resource file and store it using GeoNode's `storage_manager`. + + Downloads use the django `UploadedFile` helper classes. Depending on the size of the + remote resource, we may download it into an in-memory buffer or store it on a + temporary location on disk. After having downloaded the file, we use `storage_manager` + to save it in the appropriate location. + + """ + + response = requests.get(url, stream=True) + response.raise_for_status() + file_size = response.headers.get("Content-Length") + content_type = response.headers.get("Content-Type") + charset = response.apparent_encoding + size_threshold = config.get_setting("HARVESTED_RESOURCE_FILE_MAX_MEMORY_SIZE") + if file_size is not None and int(file_size) < size_threshold: + logger.debug("Downloading to an in-memory buffer...") + buf = io.BytesIO() + file_ = uploadedfile.InMemoryUploadedFile(buf, None, target_name, content_type, file_size, charset) + else: + logger.debug("Downloading to a temporary file...") + file_ = uploadedfile.TemporaryUploadedFile(target_name, content_type, file_size, charset) + # NOTE: there is no need to explicitly delete the file represented by + # `file_`, it is being deleted implicitly + with file_.open("wb+") as fd: + for chunk in response.iter_content(chunk_size=None, decode_unicode=False): + fd.write(chunk) + fd.seek(0) + if storage_manager.exists(target_name): + logger.debug(f"file {target_name!r} already exists, replacing...") + storage_manager.delete(target_name) + file_name = storage_manager.save(target_name, fd) + result = Path(storage_manager.path(file_name)) + return result + + +def _get_file_name( + resource_info: HarvestedResourceInfo, +) -> typing.Optional[str]: + file_extension = {"geotiff": ".tiff", "shapefile": ".zip"}.get( + resource_info.resource_descriptor.identification.native_format, + f".{resource_info.resource_descriptor.identification.native_format}", + ) + base_fragment = resource_info.resource_descriptor.identification.name + base_name = html.unescape(base_fragment).rsplit("/")[-1].rsplit("\\")[-1] + if base_name in {"", ".", ".."}: + result = None + else: + result = f"harvested_{resource_info.resource_descriptor.uuid}_{base_name}{file_extension}" + # FIXME: geonode.upload.files._clean_string ought to be renamed in order to not indicate it is private + sanitized = geonode.upload.files._clean_string(result) # noqa + return sanitized + + +def _consolidate_resource_keywords( + resource_descriptor: resourcedescriptor.RecordDescription, geonode_resource, harvester_id: int +) -> typing.List[str]: + geonode_keywords = geonode_resource.keyword_list() if geonode_resource else [] + keywords = list(resource_descriptor.identification.other_keywords) + geonode_keywords + return list(set(keywords)) diff --git a/geonode/harvesting/harvesters/geonodeharvester.py b/geonode/harvesting/harvesters/geonodeharvester.py new file mode 100644 index 0000000..dad1571 --- /dev/null +++ b/geonode/harvesting/harvesters/geonodeharvester.py @@ -0,0 +1,1289 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""Harvesters GeoNode remote servers.""" + +import datetime as dt +import enum +import json +import logging +import math +import typing +import urllib.parse +import uuid + +import dateutil.parser +import requests +from django.contrib.gis import geos +from lxml import etree + +from geonode.layers.enumerations import GXP_PTYPES +from geonode.documents.models import Document +from geonode.layers.models import Dataset +from geonode.maps.models import Map + +from .. import ( + models, + resourcedescriptor, +) +from geonode.utils import XML_PARSER, get_xpath_value +from . import base + +logger = logging.getLogger(__name__) + + +class GeoNodeDatasetType(enum.Enum): + VECTOR = "vector" + RASTER = "raster" + + +class RemoteDatasetType(enum.Enum): + VECTOR = "shapefile" + RASTER = "geotiff" + + +class GeoNodeResourceType(enum.Enum): + DOCUMENT = "documents" + DATASET = "layers" + MAP = "maps" + + +class GeoNodeResourceTypeCurrent(enum.Enum): + DOCUMENT = "document" + DATASET = "dataset" + + +class GeonodeCurrentHarvester(base.BaseHarvesterWorker): + """A harvester for modern (v3.2+) GeoNode versions. + + GeoNode versions above 3.2 introduced the concept of `datasets` to replace the older + `layers` concept. The API also has some significative differences. + + """ + + harvest_documents: bool + harvest_datasets: bool + + # harvesting of maps is explicitly disabled - the GeoNode API does not + # really allow reconstructing a Map via API, as there is no information + # about the actual contents of the map, i.e. which layers are contained + # in it + harvest_maps: bool = False + + copy_documents: bool + copy_datasets: bool + resource_title_filter: typing.Optional[str] + start_date_filter: typing.Optional[str] + end_date_filter: typing.Optional[str] + keywords_filter: typing.Optional[typing.List[str]] + categories_filter: typing.Optional[typing.List[str]] + http_session: requests.Session + page_size: int = 10 + + def __init__( + self, + *args, + harvest_documents: typing.Optional[bool] = True, + harvest_datasets: typing.Optional[bool] = True, + copy_datasets: typing.Optional[bool] = False, + copy_documents: typing.Optional[bool] = False, + resource_title_filter: typing.Optional[str] = None, + start_date_filter: typing.Optional[str] = None, + end_date_filter: typing.Optional[str] = None, + keywords_filter: typing.Optional[typing.List[str]] = None, + categories_filter: typing.Optional[typing.List[str]] = None, + **kwargs, + ): + """A harvester for remote GeoNode instances.""" + super().__init__(*args, **kwargs) + self.remote_url = self.remote_url.rstrip("/") + self.http_session = requests.Session() + self.harvest_documents = bool(harvest_documents) + self.harvest_datasets = bool(harvest_datasets) + self.copy_datasets = bool(copy_datasets) + self.copy_documents = bool(copy_documents) + self.resource_title_filter = resource_title_filter + self.start_date_filter = start_date_filter + self.end_date_filter = end_date_filter + self.keywords_filter = keywords_filter + self.categories_filter = categories_filter + + @property + def base_api_url(self): + return f"{self.remote_url}/api/v2" + + @property + def allows_copying_resources(self) -> bool: + return True + + @classmethod + def from_django_record(cls, record: models.Harvester): + return _from_django_record(cls, record) + + @classmethod + def get_extra_config_schema(cls) -> typing.Dict: + return _get_extra_config_schema() + + def get_num_available_resources(self) -> int: + url = f"{self.base_api_url}/resources/" + response = self.http_session.get(url, params=self._get_resource_list_params()) + result = 0 + if response.status_code == requests.codes.ok: + try: + result = response.json().get("total", 0) + except json.JSONDecodeError as exc: + logger.exception("Could not decode response as a JSON object") + raise base.HarvestingException(str(exc)) + else: + logger.error(f"Got back invalid response from {url!r}: {response.status_code}") + return result + + def list_resources(self, offset: typing.Optional[int] = 0) -> typing.List[base.BriefRemoteResource]: + url = f"{self.base_api_url}/resources/" + response = self.http_session.get(url, params=self._get_resource_list_params(offset)) + result = [] + if response.status_code == requests.codes.ok: + try: + payload = response.json() + except json.JSONDecodeError as exc: + logger.exception("Could not decode response as a JSON object") + raise base.HarvestingException(str(exc)) + else: + for raw_resource in payload.get("resources", []): + try: + brief_resource = base.BriefRemoteResource( + unique_identifier=raw_resource["pk"], + title=raw_resource["title"], + abstract=raw_resource["abstract"], + resource_type=raw_resource["resource_type"], + ) + result.append(brief_resource) + except KeyError as exc: + logger.exception(f"Could not decode resource: {raw_resource!r}") + raise base.HarvestingException(str(exc)) + else: + logger.error(f"Got back invalid response from {url!r}: {response.status_code}") + return result + + def check_availability(self, timeout_seconds: typing.Optional[int] = 5) -> bool: + return _check_availability(self.http_session, f"{self.base_api_url}/datasets", "datasets", timeout_seconds) + + def get_geonode_resource_type(self, remote_resource_type: str) -> typing.Type[typing.Union[Dataset, Document]]: + return {GeoNodeResourceTypeCurrent.DATASET.value: Dataset, GeoNodeResourceTypeCurrent.DOCUMENT.value: Document}[ + remote_resource_type + ] + + def get_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> typing.Optional[base.HarvestedResourceInfo]: + url_fragment = { + GeoNodeResourceTypeCurrent.DATASET.value: "/datasets/", + GeoNodeResourceTypeCurrent.DOCUMENT.value: "/documents/", + }[harvestable_resource.remote_resource_type] + url = f"{self.base_api_url}{url_fragment}{harvestable_resource.unique_identifier}/" + response = self.http_session.get(url) + result = None + if response.status_code == requests.codes.ok: + try: + response_payload = response.json() + except json.JSONDecodeError: + logger.exception("Could not decode response payload as valid JSON") + else: + resource_descriptor = self._get_resource_descriptor( + response_payload, harvestable_resource.remote_resource_type + ) + result = base.HarvestedResourceInfo( + resource_descriptor=resource_descriptor, additional_information=None + ) + else: + logger.error( + f"Could not retrieve remote resource with unique " + f"identifier {harvestable_resource.unique_identifier!r}" + ) + return result + + def should_copy_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> bool: + return { + GeoNodeResourceTypeCurrent.DATASET.value: self.copy_datasets, + GeoNodeResourceTypeCurrent.DOCUMENT.value: self.copy_documents, + }.get(harvestable_resource.remote_resource_type, False) + + def get_geonode_resource_defaults( + self, + harvested_info: base.HarvestedResourceInfo, + harvestable_resource: models.HarvestableResource, + ) -> typing.Dict: + defaults = super().get_geonode_resource_defaults(harvested_info, harvestable_resource) + defaults.update(harvested_info.resource_descriptor.additional_parameters) + local_resource_type = self.get_geonode_resource_type(harvestable_resource.remote_resource_type) + to_copy = self.should_copy_resource(harvestable_resource) + if local_resource_type == Document and not to_copy: + # since we are not copying the document, we need to provide suitable remote URLs + defaults.update( + { + "doc_url": harvested_info.resource_descriptor.distribution.embed_url, + "thumbnail_url": harvested_info.resource_descriptor.distribution.thumbnail_url, + } + ) + elif local_resource_type == Dataset: + defaults.update({"name": harvested_info.resource_descriptor.identification.name}) + if not to_copy: + # since we are not copying the dataset, we need to provide suitable SRID and remote URL + try: + srid = harvested_info.resource_descriptor.reference_systems[0] + except AttributeError: + srid = None + defaults.update( + { + "alternate": defaults["alternate"], + "workspace": defaults["workspace"], + "ows_url": harvested_info.resource_descriptor.distribution.wms_url, + "thumbnail_url": harvested_info.resource_descriptor.distribution.thumbnail_url, + "srid": srid, + "ptype": GXP_PTYPES["GN_WMS"], + "subtype": "remote", + } + ) + return defaults + + def _get_contact_descriptor(self, role, contact_details: typing.Dict): + return resourcedescriptor.RecordDescriptionContact( + role=role, name=self._get_related_name(contact_details) or contact_details["username"] + ) + + def _get_related_name(self, contact_details: typing.Dict): + return " ".join((contact_details.get("first_name", ""), contact_details.get("last_name", ""))).strip() + + def _get_document_link_info(self, resource: typing.Dict): + native_format = resource["extension"] + download_url = resource["href"] + return native_format, download_url + + def _get_dataset_link_info(self, resource: typing.Dict, spatial_extent: geos.Polygon): + wms_url = None + wfs_url = None + wcs_url = None + download_url = None + native_format = None + from geonode.geoserver.ows import _wfs_link, _wcs_link + + for link_info in resource.get("links", []): + type_ = link_info["link_type"] + if type_ == "OGC:WMS": + wms_url = link_info["url"] + elif type_ == "OGC:WFS": + wfs_url = link_info["url"] + native_format = "shapefile" + + download_url = _wfs_link( + wfs_url, + resource["name"], + "SHAPE-ZIP", + {"format_options": "charset:UTF-8"}, + srid=resource["srid"], + bbox=None, + ) + elif type_ == "OGC:WCS": + wcs_url = link_info["url"] + native_format = "geotiff" + coords = spatial_extent.coords[0] + min_x = min([i[0] for i in coords]) + max_x = max([i[0] for i in coords]) + min_y = min([i[1] for i in coords]) + max_y = max([i[1] for i in coords]) + + download_url = _wcs_link( + wcs_url, + resource["alternate"], + "image/tiff", + srid=resource["srid"], + bbox=[min_x, min_y, max_x, max_y], + compression="DEFLATE", + tile_size=512, + ) + return native_format, download_url, wms_url, wfs_url, wcs_url + + def _get_resource_link_info( + self, + resource: typing.Dict, + remote_resource_type: str, + spatial_extent: geos.Polygon, + ) -> typing.Tuple[str, str, str, str, typing.Optional[str], typing.Optional[str], typing.Optional[str]]: + embed_url = resource["embed_url"] + thumbnail_url = resource["thumbnail_url"] + if remote_resource_type == GeoNodeResourceTypeCurrent.DATASET.value: + native_format, download_url, wms_url, wfs_url, wcs_url = self._get_dataset_link_info( + resource, spatial_extent + ) + else: + wms_url = None + wfs_url = None + wcs_url = None + native_format, download_url = self._get_document_link_info(resource) + return native_format, download_url, embed_url, thumbnail_url, wms_url, wfs_url, wcs_url + + def _get_resource_descriptor( + self, + raw_resource: typing.Dict, + remote_resource_type: str, + ) -> resourcedescriptor.RecordDescription: + resource = raw_resource[remote_resource_type] + resource_date = dateutil.parser.isoparse(resource["date"]) + resource_datestamp = dateutil.parser.isoparse(resource["last_updated"]) + try: # these are sometimes returned as None + temporal_extent_start = dateutil.parser.isoparse(resource["temporal_extent_start"]) + temporal_extent_end = dateutil.parser.isoparse(resource["temporal_extent_end"]) + except TypeError: + temporal_extent_start = None + temporal_extent_end = None + spatial_extent = geos.GEOSGeometry(json.dumps(resource.get("bbox_polygon"))) + lonlat_extent = geos.GEOSGeometry(json.dumps(resource.get("ll_bbox_polygon"))) + link_info = self._get_resource_link_info(resource, remote_resource_type, spatial_extent) + native_format, download_url, embed_url, thumbnail_url, wms_url, wfs_url, wcs_url = link_info + descriptor = resourcedescriptor.RecordDescription( + # these work for both datasets and documents + uuid=resource["uuid"], + language=resource["language"], + # TODO issue#10290 + point_of_contact=self._get_contact_descriptor("pointOfContact", resource["poc"]), + author=self._get_contact_descriptor("author", resource["metadata_author"]), + date_stamp=resource_datestamp, + reference_systems=[resource["srid"]], + data_quality=resource.get("raw_data_quality_statement"), + character_set=resource.get("charset", "UTF-8"), + identification=resourcedescriptor.RecordIdentification( + name=resource["name"], + title=resource["title"], + date=resource_date, + date_type=resource["date_type"], + originator=self._get_contact_descriptor("originator", resource["owner"]), + place_keywords=[i.get("code") for i in resource.get("regions", [])], + other_keywords=[i.get("slug") for i in resource.get("keywords", [])], + license=(resource.get("license") or {}).get("identifier"), + abstract=resource.get("raw_abstract", ""), + purpose=resource.get("raw_purpose", ""), + native_format=native_format, + other_constraints=resource.get("raw_constraints_other"), + topic_category=(resource.get("category") or {}).get("identifier"), + supplemental_information=resource.get("raw_supplemental_information"), + spatial_extent=spatial_extent, + lonlat_extent=lonlat_extent, + temporal_extent=(temporal_extent_start, temporal_extent_end) if temporal_extent_start else None, + ), + distribution=resourcedescriptor.RecordDistribution( + link_url=resource["link"], + wms_url=wms_url, + wfs_url=wfs_url, + wcs_url=wcs_url, + thumbnail_url=thumbnail_url, + download_url=download_url, + embed_url=embed_url, + ), + additional_parameters={ + "subtype": resource["subtype"], + "resource_type": remote_resource_type, + }, + ) + if remote_resource_type == GeoNodeResourceTypeCurrent.DOCUMENT.value: + descriptor.additional_parameters["extension"] = resource["extension"] + elif remote_resource_type == GeoNodeResourceTypeCurrent.DATASET.value: + descriptor.additional_parameters.update( + { + "alternate": resource["alternate"], + "workspace": resource["workspace"], + } + ) + return descriptor + + def _get_resource_list_params(self, offset: typing.Optional[int] = 0) -> typing.Dict: + current_page = math.floor((offset + self.page_size) / self.page_size) + result = { + "page_size": self.page_size, + "page": current_page, + } + resource_filter = [] + if self.harvest_datasets: + resource_filter.append(GeoNodeResourceTypeCurrent.DATASET.value) + if self.harvest_documents: + resource_filter.append(GeoNodeResourceTypeCurrent.DOCUMENT.value) + if len(resource_filter) > 0: + result["filter{resource_type.in}"] = resource_filter + + if self.resource_title_filter is not None: + result["filter{title.icontains}"] = self.resource_title_filter + if self.start_date_filter is not None: + start_date = dateutil.parser.parse(self.start_date_filter) + result["filter{date.gte}"] = ( + f"{start_date.astimezone(dt.timezone.utc).replace(microsecond=0).isoformat().split('+')[0]}Z" + ) + if self.end_date_filter is not None: + end_date = dateutil.parser.parse(self.end_date_filter) + result["filter{date.lte}"] = ( + f"{end_date.astimezone(dt.timezone.utc).replace(microsecond=0).isoformat().split('+')[0]}Z" + ) + if self.keywords_filter is not None: + result["filter{keywords.slug.in}"] = self.keywords_filter + if self.categories_filter is not None: + result["filter{category.identifier.in}"] = self.categories_filter + return result + + +class GeonodeLegacyHarvester(base.BaseHarvesterWorker): + """A harvester for older (v <= 3.2) GeoNode versions""" + + harvest_documents: bool + harvest_datasets: bool + + # harvesting of maps is explicitly disabled - the GeoNode API does not + # really allow reconstructing a Map via API, as there is no information + # about the actual contents of the map, i.e. which layers are contained + # in it + harvest_maps: bool = False + + copy_documents: bool + copy_datasets: bool + resource_title_filter: typing.Optional[str] + http_session: requests.Session + page_size: int = 10 + + def __init__( + self, + *args, + harvest_documents: typing.Optional[bool] = True, + harvest_datasets: typing.Optional[bool] = True, + copy_datasets: typing.Optional[bool] = False, + copy_documents: typing.Optional[bool] = False, + resource_title_filter: typing.Optional[str] = None, + start_date_filter: typing.Optional[str] = None, + end_date_filter: typing.Optional[str] = None, + keywords_filter: typing.Optional[list] = None, + categories_filter: typing.Optional[list] = None, + **kwargs, + ): + """A harvester for remote GeoNode instances.""" + super().__init__(*args, **kwargs) + self.remote_url = self.remote_url.rstrip("/") + self.http_session = requests.Session() + self.harvest_documents = harvest_documents if harvest_documents is not None else True + self.harvest_datasets = harvest_datasets if harvest_datasets is not None else True + self.copy_datasets = copy_datasets + self.copy_documents = copy_documents + self.resource_title_filter = resource_title_filter + self.start_date_filter = start_date_filter + self.end_date_filter = end_date_filter + self.keywords_filter = keywords_filter + self.categories_filter = categories_filter + + @property + def base_api_url(self): + return f"{self.remote_url}/api" + + @property + def allows_copying_resources(self) -> bool: + return True + + @classmethod + def from_django_record(cls, record: models.Harvester): + return _from_django_record(cls, record) + + @classmethod + def get_extra_config_schema(cls) -> typing.Dict: + return _get_extra_config_schema() + + def get_num_available_resources(self) -> int: + result = 0 + if self.harvest_datasets: + result += self._get_total_records(GeoNodeResourceType.DATASET) + if self.harvest_documents: + result += self._get_total_records(GeoNodeResourceType.DOCUMENT) + if self.harvest_maps: + result += self._get_total_records(GeoNodeResourceType.MAP) + return result + + def list_resources(self, offset: typing.Optional[int] = 0) -> typing.List[base.BriefRemoteResource]: + # the implementation of this method is a bit convoluted because the GeoNode + # legacy versions do not have a common `/resources` endpoint, we must query + # the individual endpoints for documents, layers and maps and work out the + # correct offsets to use. + total_resources = self._get_num_available_resources_by_type() + if offset < total_resources[GeoNodeResourceType.DOCUMENT]: + document_list = self._list_document_resources(offset) + if len(document_list) < self.page_size: + dataset_list = self._list_dataset_resources(0) + added = document_list + dataset_list + if len(added) < self.page_size: + map_list = self._list_map_resources(0) + result = (added + map_list)[: self.page_size] + else: + result = added[: self.page_size] + else: + result = document_list + elif offset < (total_resources[GeoNodeResourceType.DOCUMENT] + total_resources[GeoNodeResourceType.DATASET]): + dataset_offset = offset - total_resources[GeoNodeResourceType.DOCUMENT] + dataset_list = self._list_dataset_resources(dataset_offset) + if len(dataset_list) < self.page_size: + map_list = self._list_map_resources(0) + result = (dataset_list + map_list)[: self.page_size] + else: + result = dataset_list + else: + map_offset = offset - ( + total_resources[GeoNodeResourceType.DOCUMENT] + total_resources[GeoNodeResourceType.DATASET] + ) + result = self._list_map_resources(map_offset) + return result + + def check_availability(self, timeout_seconds: typing.Optional[int] = 5) -> bool: + """Check whether the remote GeoNode is online.""" + return _check_availability(self.http_session, f"{self.base_api_url}/", "layers", timeout_seconds) + + def get_geonode_resource_type(self, remote_resource_type: str) -> typing.Type[typing.Union[Dataset, Document, Map]]: + """Return resource type class from resource type string.""" + return { + GeoNodeResourceType.MAP.value: Map, + GeoNodeResourceType.DATASET.value: Dataset, + GeoNodeResourceType.DOCUMENT.value: Document, + }[remote_resource_type] + + def get_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> typing.Optional[base.HarvestedResourceInfo]: + resource_unique_identifier = harvestable_resource.unique_identifier + local_resource_type = self.get_geonode_resource_type(harvestable_resource.remote_resource_type) + endpoint_suffix = { + Document: (f"/documents/{resource_unique_identifier}/"), + Dataset: f"/layers/{resource_unique_identifier}/", + Map: f"/maps/{resource_unique_identifier}/", + }[local_resource_type] + url = f"{self.base_api_url}{endpoint_suffix}" + response = self.http_session.get(url) + result = None + if response.status_code == requests.codes.ok: + api_record = response.json() + resource_descriptor = self._get_resource_details(api_record, harvestable_resource) + result = base.HarvestedResourceInfo(resource_descriptor=resource_descriptor, additional_information=None) + else: + logger.error( + f"Could not retrieve remote resource with unique " f"identifier {resource_unique_identifier!r}" + ) + return result + + def should_copy_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> bool: + return { + GeoNodeResourceType.DOCUMENT.value: self.copy_documents, + GeoNodeResourceType.DATASET.value: self.copy_datasets, + GeoNodeResourceType.MAP.value: False, + }[harvestable_resource.remote_resource_type] + + def get_geonode_resource_defaults( + self, + harvested_info: base.HarvestedResourceInfo, + harvestable_resource: models.HarvestableResource, + ) -> typing.Dict: + defaults = super().get_geonode_resource_defaults(harvested_info, harvestable_resource) + defaults.update(harvested_info.resource_descriptor.additional_parameters) + local_resource_type = self.get_geonode_resource_type(harvestable_resource.remote_resource_type) + to_copy = self.should_copy_resource(harvestable_resource) + if local_resource_type == Document and not to_copy: + # since we are not copying the document, we need to provide suitable remote URLs + defaults.update( + { + "doc_url": harvested_info.resource_descriptor.distribution.download_url, + "thumbnail_url": harvested_info.resource_descriptor.distribution.thumbnail_url, + } + ) + elif local_resource_type == Dataset: + defaults.update( + { + "name": harvested_info.resource_descriptor.identification.name, + } + ) + if not to_copy: + # since we are not copying the dataset, we need to provide suitable SRID and remote URL + try: + srid = harvested_info.resource_descriptor.reference_systems[0] + except AttributeError: + srid = None + defaults.update( + { + "name": defaults["name"].rpartition(":")[-1], + "ows_url": harvested_info.resource_descriptor.distribution.wms_url, + "thumbnail_url": harvested_info.resource_descriptor.distribution.thumbnail_url, + "srid": srid, + } + ) + return defaults + + def _get_num_available_resources_by_type(self) -> typing.Dict[GeoNodeResourceType, int]: + result = { + GeoNodeResourceType.DOCUMENT: 0, + GeoNodeResourceType.DATASET: 0, + GeoNodeResourceType.MAP: 0, + } + if self.harvest_documents: + result[GeoNodeResourceType.DOCUMENT] = self._get_total_records(GeoNodeResourceType.DOCUMENT) + if self.harvest_datasets: + result[GeoNodeResourceType.DATASET] = self._get_total_records(GeoNodeResourceType.DATASET) + if self.harvest_maps: + result[GeoNodeResourceType.MAP] = self._get_total_records(GeoNodeResourceType.MAP) + return result + + def _list_document_resources(self, offset: int) -> typing.List[base.BriefRemoteResource]: + result = [] + if self.harvest_documents: + result = self._list_resources_by_type(GeoNodeResourceType.DOCUMENT, offset) + return result + + def _list_dataset_resources(self, offset: int) -> typing.List[base.BriefRemoteResource]: + result = [] + if self.harvest_datasets: + result = self._list_resources_by_type(GeoNodeResourceType.DATASET, offset) + return result + + def _list_map_resources(self, offset: int) -> typing.List[base.BriefRemoteResource]: + result = [] + if self.harvest_maps: + result = self._list_resources_by_type(GeoNodeResourceType.MAP, offset) + return result + + def _list_resources_by_type( + self, resource_type: GeoNodeResourceType, offset: int + ) -> typing.List[base.BriefRemoteResource]: + response = self.http_session.get( + f"{self.base_api_url}/{resource_type.value}/", params=self._get_resource_list_params(offset) + ) + response.raise_for_status() + result = [] + for resource in response.json().get("objects", []): + brief_resource = base.BriefRemoteResource( + unique_identifier=self._extract_unique_identifier(resource), + title=resource["title"], + abstract=resource["abstract"], + resource_type=resource_type.value, + ) + result.append(brief_resource) + return result + + def _extract_unique_identifier(self, raw_remote_resource: typing.Dict) -> str: + return raw_remote_resource["id"] + + def _get_resource_details( + self, api_record: typing.Dict, harvestable_resource: models.HarvestableResource + ) -> typing.Optional[resourcedescriptor.RecordDescription]: + """ + Produce a record description from the response provided by the remote GeoNode. + """ + result = None + # query CSW endpoint and parse its response in order to get more information + # about the resource + get_record_by_id_response = self.http_session.get( + f"{self.remote_url}/catalogue/csw", + params={ + "service": "CSW", + "version": "2.0.2", + "request": "GetRecordById", + "id": api_record["uuid"], + "elementsetname": "full", + "outputschema": "http://www.isotc211.org/2005/gmd", + }, + ) + if get_record_by_id_response.status_code == requests.codes.ok: + xml_root = etree.fromstring(get_record_by_id_response.content, parser=XML_PARSER) + try: + metadata_element = xml_root.xpath("./gmd:MD_Metadata", namespaces=xml_root.nsmap)[0] + except IndexError: + logger.warning("Unable to retrieve a metadata element from the CSW GetRecordById response, skipping...") + logger.debug(f"Original response content: {get_record_by_id_response.content}") + else: + try: + result = self._get_resource_descriptor(metadata_element, api_record, harvestable_resource) + except TypeError: + logger.exception("Could not retrieve metadata details to generate resource descriptor, skipping...") + else: + logger.debug(f"Found details for resource {result.uuid!r} - {result.identification.title!r}") + else: + logger.warning(f"Got invalid response from {get_record_by_id_response.request.url}") + return result + + def _get_resource_list_params(self, offset: typing.Optional[int] = 0) -> typing.Dict: + result = { + "limit": self.page_size, + "offset": offset, + } + if self.resource_title_filter is not None: + result["title__icontains"] = self.resource_title_filter + if self.start_date_filter is not None: + start_date = dateutil.parser.parse(self.start_date_filter) + result["date__gte"] = ( + f"{start_date.astimezone(dt.timezone.utc).replace(microsecond=0).isoformat().split('+')[0]}Z" + ) + if self.end_date_filter is not None: + end_date = dateutil.parser.parse(self.end_date_filter) + result["date__lte"] = ( + f"{end_date.astimezone(dt.timezone.utc).replace(microsecond=0).isoformat().split('+')[0]}Z" + ) + if self.keywords_filter is not None: + result["keywords__slug__in"] = ",".join(self.keywords_filter) + if self.categories_filter is not None: + result["category__identifier__in"] = ",".join(self.categories_filter) + return result + + def _get_total_records( + self, + resource_type: GeoNodeResourceType, + ) -> int: + url = f"{self.base_api_url}/{resource_type.value}/" + response = self.http_session.get(url, params=self._get_resource_list_params()) + result = 0 + if response.status_code != requests.codes.ok: + logger.error(f"Got back invalid response from {url!r}") + else: + try: + result = response.json().get("meta", {}).get("total_count", 0) + except json.JSONDecodeError as exc: + logger.exception("Could not decode response as a JSON object") + raise base.HarvestingException(str(exc)) + return result + + def _get_resource_descriptor( + self, csw_record: etree.Element, api_record: typing.Dict, harvestable_resource: models.HarvestableResource + ) -> resourcedescriptor.RecordDescription: + identification_descriptor = get_identification_descriptor( + csw_record.xpath("gmd:identificationInfo", namespaces=csw_record.nsmap)[0], api_record + ) + crs = api_record.get("srid", "EPSG:4326") + descriptor = resourcedescriptor.RecordDescription( + uuid=uuid.UUID(get_xpath_value(csw_record, "gmd:fileIdentifier")), + language=get_xpath_value(csw_record, "gmd:language"), + character_set=csw_record.xpath( + "gmd:characterSet/gmd:MD_CharacterSetCode/text()", namespaces=csw_record.nsmap + )[0], + point_of_contact=get_contact_descriptor( + csw_record.xpath( + "gmd:contact[.//gmd:role//@codeListValue='pointOfContact']", namespaces=csw_record.nsmap + )[0] + ), + author=get_contact_descriptor( + csw_record.xpath("gmd:contact[.//gmd:role//@codeListValue='author']", namespaces=csw_record.nsmap)[0] + ), + date_stamp=dateutil.parser.parse( + csw_record.xpath("gmd:dateStamp/gco:DateTime/text()", namespaces=csw_record.nsmap)[0], + ).replace(tzinfo=dt.timezone.utc), + reference_systems=[crs], + identification=identification_descriptor, + distribution=self.get_distribution_info( + csw_record.xpath("gmd:distributionInfo", namespaces=csw_record.nsmap)[0], + api_record, + harvestable_resource, + identification_descriptor, + crs, + ), + data_quality=get_xpath_value(csw_record, ".//gmd:dataQualityInfo//gmd:lineage"), + ) + additional_params_handler = { + GeoNodeResourceType.DATASET.value: self._get_dataset_additional_parameters, + GeoNodeResourceType.DOCUMENT.value: self._get_document_additional_parameters, + GeoNodeResourceType.MAP.value: self._get_map_additional_parameters, + }[harvestable_resource.remote_resource_type] + additional_params = additional_params_handler(descriptor, api_record) + descriptor.additional_parameters.update(additional_params) + return descriptor + + def _get_dataset_additional_parameters( + self, descriptor: resourcedescriptor.RecordDescription, api_record: typing.Dict + ) -> typing.Dict: + result = { + "name": descriptor.identification.name, + "charset": descriptor.character_set, + "resource_type": "dataset", + "alternate": api_record.get("alternate", descriptor.identification.name), + "workspace": api_record.get("workspace"), + } + if descriptor.identification.native_format.lower() == RemoteDatasetType.VECTOR.value: + result["subtype"] = GeoNodeDatasetType.VECTOR.value + elif descriptor.identification.native_format.lower() == RemoteDatasetType.RASTER.value: + result["subtype"] = GeoNodeDatasetType.RASTER.value + return result + + def _get_document_additional_parameters( + self, descriptor: resourcedescriptor.RecordDescription, api_record: typing.Dict + ) -> typing.Dict: + return {"resource_type": "document", "extension": api_record.get("extension")} + + def _get_map_additional_parameters( + self, descriptor: resourcedescriptor.RecordDescription, api_record: typing.Dict + ) -> typing.Dict: + return {"last_modified": api_record.get("last_modified")} + + def get_distribution_info( + self, + csw_distribution: etree.Element, + api_record: typing.Dict, + harvestable_resource: models.HarvestableResource, + identification_descriptor: resourcedescriptor.RecordIdentification, + crs: str, + ) -> resourcedescriptor.RecordDistribution: + online_elements = csw_distribution.xpath( + ".//gmd:transferOptions//gmd:onLine", namespaces=csw_distribution.nsmap + ) + link = None + wms = None + wfs = None + wcs = None + original = None + original_format_values = ( + "original dataset format", + "hosted document format", + ) + for online_el in online_elements: + protocol = get_xpath_value(online_el, ".//gmd:protocol").lower() + linkage = get_xpath_value(online_el, ".//gmd:linkage") + description = (get_xpath_value(online_el, ".//gmd:description") or "").lower() + if "link" in protocol: + link = linkage + elif "ogc:wms" in protocol: + wms = linkage + elif "ogc:wfs" in protocol: + wfs = linkage + elif "ogc:wcs" in protocol: + wcs = linkage + else: + for original_value in original_format_values: + if original_value in description.lower(): + original = linkage + break + if harvestable_resource.remote_resource_type == GeoNodeResourceType.DATASET.value: + # for layers, we generate a download URL for a zipped shapefile, in a similar way + # as is done on the main GeoNode UI, by leveraging WFS + if wfs is not None: + query_params = { + "service": "WFS", + "version": "1.0.0", + "request": "GetFeature", + "typename": identification_descriptor.name, + "outputformat": "SHAPE-ZIP", + "srs": crs, + "format_options": "charset:UTF-8", + } + original = f"{wfs}?{urllib.parse.urlencode(query_params)}" + elif wcs is not None: + coords = identification_descriptor.spatial_extent.coords[0] + min_x = min([i[0] for i in coords]) + max_x = max([i[0] for i in coords]) + min_y = min([i[1] for i in coords]) + max_y = max([i[1] for i in coords]) + query_params = { + "service": "WCS", + "version": "2.0.1", + "request": "GetCoverage", + "srs": crs, + "format": "image/tiff", + "coverageid": api_record["alternate"].replace(":", "__"), + "bbox": f"{min_x},{min_y},{max_x},{max_y}", + } + original = f"{wcs}?{urllib.parse.urlencode(query_params)}" + else: + try: + original = [ + record_link.get("url") + for record_link in api_record.get("links", []) + if record_link.get("name") == "Zipped Shapefile" + ][0] + except IndexError: + pass + return resourcedescriptor.RecordDistribution( + link_url=link, + wms_url=wms, + wfs_url=wfs, + wcs_url=wcs, + thumbnail_url=self._retrieve_thumbnail_url(api_record, harvestable_resource), + download_url=original, + embed_url=original, + ) + + def _retrieve_thumbnail_url( + self, + api_record: typing.Dict, + harvestable_resource: models.HarvestableResource, + ) -> str: + is_document = harvestable_resource.remote_resource_type == GeoNodeResourceType.DOCUMENT.value + found_thumbnail = False + if is_document: + # there seems to be a bug in GeoNode whereby the thumbnail for documents + # is reported correctly on the document list endpoint but it is reported + # incorrectly on the document detail endpoint and it is also incorrect in + # the CSW response. Therefore we resort to fetching the thumbnail through + # the document list endpoint + document_list_response = self.http_session.get( + f"{self.base_api_url}/documents/", params={"id": harvestable_resource.unique_identifier} + ) + if document_list_response.status_code == requests.codes.ok: + raw_response = document_list_response.json() + try: + reported_thumbnail: str = raw_response["objects"][0]["thumbnail_url"] + except (KeyError, IndexError): + logger.exception("Could not retrieve document details from the list endpoint") + else: + if reported_thumbnail.startswith("/"): + thumbnail = f"{self.remote_url}{reported_thumbnail}" + else: + thumbnail = reported_thumbnail + found_thumbnail = True + else: + logger.debug( + f"Could not retrieve document list endpoint: {document_list_response.status_code} - {document_list_response.reason}" + ) + if not found_thumbnail: + reported_thumbnail: typing.Optional[str] = api_record.get("thumbnail_url") + if reported_thumbnail is not None and reported_thumbnail.startswith("/"): + # it is not a full URL + thumbnail = f"{self.remote_url}{reported_thumbnail}" + else: + thumbnail = reported_thumbnail + return thumbnail + + +class GeonodeUnifiedHarvesterWorker(base.BaseHarvesterWorker): + """A harvester worker that is able to retrieve details from most GeoNode deployments. + + This harvester type relies on the `GeonodeCurrentHarvester` and `GeonodeLegacyHarvester` for most + operations. It simply determines which concrete harvester to use based on the remote's response + for the availability check and then uses it. + + """ + + _concrete_harvester_worker: typing.Optional[typing.Union[GeonodeCurrentHarvester, GeonodeLegacyHarvester]] + + def __init__( + self, + *args, + harvest_documents: typing.Optional[bool] = True, + harvest_datasets: typing.Optional[bool] = True, + copy_datasets: typing.Optional[bool] = False, + copy_documents: typing.Optional[bool] = False, + resource_title_filter: typing.Optional[str] = None, + start_date_filter: typing.Optional[str] = None, + end_date_filter: typing.Optional[str] = None, + keywords_filter: typing.Optional[typing.List[str]] = None, + categories_filter: typing.Optional[typing.List[str]] = None, + **kwargs, + ): + """A harvester for remote GeoNode instances.""" + self._concrete_harvester_worker = None + super().__init__(*args, **kwargs) + self.remote_url = self.remote_url.rstrip("/") + self.http_session = requests.Session() + self.harvest_documents = bool(harvest_documents) + self.harvest_datasets = bool(harvest_datasets) + self.copy_datasets = bool(copy_datasets) + self.copy_documents = bool(copy_documents) + self.resource_title_filter = resource_title_filter + self.start_date_filter = start_date_filter + self.end_date_filter = end_date_filter + self.keywords_filter = keywords_filter + self.categories_filter = categories_filter + + @property + def concrete_worker(self) -> typing.Union[GeonodeCurrentHarvester, GeonodeLegacyHarvester]: + if self._concrete_harvester_worker is None: + self._concrete_harvester_worker = self._get_concrete_worker() + return self._concrete_harvester_worker + + @property + def allows_copying_resources(self) -> bool: + return self.concrete_worker.allows_copying_resources + + @classmethod + def from_django_record(cls, record: models.Harvester): + return _from_django_record(cls, record) + + @classmethod + def get_extra_config_schema(cls) -> typing.Dict: + return _get_extra_config_schema() + + def get_num_available_resources(self) -> int: + return self.concrete_worker.get_num_available_resources() + + def list_resources(self, offset: typing.Optional[int] = 0) -> typing.List[base.BriefRemoteResource]: + return self.concrete_worker.list_resources(offset) + + def check_availability(self, timeout_seconds: typing.Optional[int] = 5) -> bool: + return self.concrete_worker.check_availability(timeout_seconds) + + def get_geonode_resource_type(self, remote_resource_type: str) -> typing.Type[typing.Union[Dataset, Document]]: + return self.concrete_worker.get_geonode_resource_type(remote_resource_type) + + def get_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> typing.Optional[base.HarvestedResourceInfo]: + return self.concrete_worker.get_resource(harvestable_resource) + + def should_copy_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> bool: + return self.concrete_worker.should_copy_resource(harvestable_resource) + + def get_geonode_resource_defaults( + self, + harvested_info: base.HarvestedResourceInfo, + harvestable_resource: models.HarvestableResource, + ) -> typing.Dict: + return self.concrete_worker.get_geonode_resource_defaults(harvested_info, harvestable_resource) + + def _get_concrete_worker(self) -> typing.Union[GeonodeCurrentHarvester, GeonodeLegacyHarvester]: + # first try to initialize the GeonodeCurrentHarvester. If not available, fall back to the legacy one + kwargs = { + "remote_url": self.remote_url, + "harvester_id": self.harvester_id, + "harvest_documents": self.harvest_documents, + "harvest_datasets": self.harvest_datasets, + "copy_documents": self.copy_documents, + "copy_datasets": self.copy_datasets, + "resource_title_filter": self.resource_title_filter, + "start_date_filter": self.start_date_filter, + "end_date_filter": self.end_date_filter, + "keywords_filter": self.keywords_filter, + "categories_filter": self.categories_filter, + } + current = GeonodeCurrentHarvester(**kwargs) + return current if current.check_availability() else GeonodeLegacyHarvester(**kwargs) + + +def get_contact_descriptor(contact: etree.Element): + return resourcedescriptor.RecordDescriptionContact( + role=contact.xpath(".//gmd:role/gmd:CI_RoleCode/text()", namespaces=contact.nsmap)[0], + name=_get_optional_attribute_value(contact, ".//gmd:individualName"), + organization=_get_optional_attribute_value(contact, ".//gmd:organisationName"), + position=_get_optional_attribute_value(contact, ".//gmd:positionName"), + phone_voice=_get_optional_attribute_value(contact, ".//gmd:contactInfo//gmd:phone//gmd:voice"), + phone_facsimile=_get_optional_attribute_value(contact, ".//gmd:contactInfo//gmd:phone//gmd:facsimile"), + address_delivery_point=_get_optional_attribute_value( + contact, ".//gmd:contactInfo//gmd:address//gmd:deliveryPoint" + ), + address_city=_get_optional_attribute_value(contact, ".//gmd:contactInfo//gmd:address//gmd:city"), + address_administrative_area=_get_optional_attribute_value( + contact, ".//gmd:contactInfo//gmd:address//gmd:administrativeArea" + ), + address_postal_code=_get_optional_attribute_value(contact, ".//gmd:contactInfo//gmd:address//gmd:postalCode"), + address_country=_get_optional_attribute_value(contact, ".//gmd:contactInfo//gmd:address//gmd:country"), + address_email=get_xpath_value(contact, ".//gmd:contactInfo//gmd:address//gmd:electronicMailAddress"), + ) + + +def get_identification_descriptor(csw_identification: etree.Element, api_record: typing.Dict): + place_keywords = csw_identification.xpath( + ".//gmd:descriptiveKeywords//gmd:keyword//text()[../../../gmd:type//" "@codeListValue='place']", + namespaces=csw_identification.nsmap, + ) + other_keywords = csw_identification.xpath( + ".//gmd:descriptiveKeywords//gmd:keyword//text()[../../../gmd:type//" "@codeListValue!='place']", + namespaces=csw_identification.nsmap, + ) + license_info = "".join( + csw_identification.xpath( + ".//gmd:resourceConstraints//gmd:otherConstraints//text()[../../..//" + "gmd:useConstraints//@codeListValue='license']", + namespaces=csw_identification.nsmap, + ) + ).strip() + other_constraints_description = "".join( + csw_identification.xpath( + ".//gmd:resourceConstraints//gmd:otherConstraints//text()[../../..//" + "gmd:useConstraints//@codeListValue!='license']", + namespaces=csw_identification.nsmap, + ) + ).strip() + other_constraints_code = ( + "".join( + csw_identification.xpath( + ".//gmd:resourceConstraints//gmd:useConstraints//text()[../../..//" + "gmd:useConstraints//@codeListValue!='license']", + namespaces=csw_identification.nsmap, + ) + ).strip() + or None + ) + name = get_xpath_value(csw_identification, ".//gmd:citation//gmd:name") + title = get_xpath_value(csw_identification, ".//gmd:citation//gmd:title") + return resourcedescriptor.RecordIdentification( + name=name or title, + title=title, + date=dateutil.parser.parse(get_xpath_value(csw_identification, ".//gmd:citation//gmd:date//gmd:date")).replace( + tzinfo=dt.timezone.utc + ), + date_type=get_xpath_value(csw_identification, ".//gmd:citation//gmd:dateType"), + abstract=get_xpath_value(csw_identification, ".//gmd:abstract"), + purpose=get_xpath_value(csw_identification, ".//gmd:purpose"), + status=get_xpath_value(csw_identification, ".//gmd:status"), + originator=get_contact_descriptor( + csw_identification.xpath(".//gmd:pointOfContact", namespaces=csw_identification.nsmap)[0] + ), + native_format=_get_native_format(csw_identification, api_record), + place_keywords=place_keywords, + other_keywords=other_keywords, + license=license_info.split(":", maxsplit=1), + other_constraints=(f"{other_constraints_code or ''}:{other_constraints_description}"), + topic_category=get_xpath_value(csw_identification, ".//gmd:topicCategory"), + spatial_extent=get_spatial_extent_native(api_record), + temporal_extent=get_temporal_extent(csw_identification), + supplemental_information=get_xpath_value(csw_identification, ".//gmd:supplumentalInformation"), + ) + + +def _get_native_format(csw_identification: etree.Element, api_record: typing.Dict) -> typing.Optional[str]: + if api_record.get("csw_type") == "document": + result = api_record.get("extension") + else: + store_type = api_record.get("storeType", "").lower() + if store_type == "coveragestore": + result = "geotiff" + elif store_type == "datastore": + result = "shapefile" + else: + result = get_xpath_value(csw_identification, ".//gmd:resourceFormat//gmd:name") + return result + + +def get_spatial_extent_4326(identification_el: etree.Element) -> typing.Optional[geos.Polygon]: + try: + extent_el = identification_el.xpath(".//gmd:extent//gmd:geographicElement", namespaces=identification_el.nsmap)[ + 0 + ] + left_x = get_xpath_value(extent_el, ".//gmd:westBoundLongitude") + right_x = get_xpath_value(extent_el, ".//gmd:eastBoundLongitude") + lower_y = get_xpath_value(extent_el, ".//gmd:southBoundLatitude") + upper_y = get_xpath_value(extent_el, ".//gmd:northBoundLatitude") + # GeoNode seems to have a bug whereby sometimes the reported extent uses a + # comma as the decimal separator, other times it uses a dot + result = geos.Polygon.from_bbox( + ( + float(left_x.replace(",", ".")), + float(lower_y.replace(",", ".")), + float(right_x.replace(",", ".")), + float(upper_y.replace(",", ".")), + ) + ) + except IndexError: + result = None + return result + + +def get_spatial_extent_native(api_record: typing.Dict): + declared_ewkt = api_record.get("bbox_polygon") + if declared_ewkt is not None: + # this is a more recent GeoNode deployment + result = geos.Polygon.from_ewkt(declared_ewkt) + else: + # this is an older GeoNode deployment + left_x = float(api_record.get("bbox_x0", 0)) + right_x = float(api_record.get("bbox_x1", 0)) + lower_y = float(api_record.get("bbox_y0", 0)) + upper_y = float(api_record.get("bbox_y1", 0)) + result = geos.Polygon.from_bbox((left_x, lower_y, right_x, upper_y)) + return result + + +def get_temporal_extent(identification_el: etree.Element) -> typing.Optional[typing.Tuple[dt.datetime, dt.datetime]]: + try: + extent_el = identification_el.xpath(".//gmd:extent//gmd:temporalElement", namespaces=identification_el.nsmap)[0] + begin = dateutil.parser.parse(get_xpath_value(extent_el, ".//gml:beginPosition")).replace( + tzinfo=dt.timezone.utc + ) + end = dateutil.parser.parse(get_xpath_value(extent_el, ".//gml:endPosition")).replace(tzinfo=dt.timezone.utc) + result = (begin, end) + except IndexError: + result = None + return result + + +def _get_optional_attribute_value(element: etree.Element, xpath: str) -> typing.Optional[str]: + return element.xpath(f"{xpath}/text()", namespaces=element.nsmap)[0].strip() or None + + +def _get_extra_config_schema() -> typing.Dict: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": ("https://geonode.org/harvesting/geonode-legacy-harvester.schema.json"), + "title": "GeoNode harvester config", + "description": ("A jsonschema for validating configuration option for GeoNode's " "remote GeoNode harvester"), + "type": "object", + "properties": { + "harvest_documents": {"type": "boolean", "default": True}, + "copy_documents": {"type": "boolean", "default": False}, + "harvest_datasets": {"type": "boolean", "default": True}, + "copy_datasets": {"type": "boolean", "default": False}, + "resource_title_filter": { + "type": "string", + }, + "start_date_filter": {"type": "string", "format": "date-time"}, + "end_date_filter": {"type": "string", "format": "date-time"}, + "keywords_filter": {"type": "array", "items": {"type": "string"}}, + "categories_filter": {"type": "array", "items": {"type": "string"}}, + }, + "additionalProperties": False, + } + + +def _from_django_record(target_class: typing.Type, record: models.Harvester): + return target_class( + record.remote_url, + record.id, + harvest_documents=record.harvester_type_specific_configuration.get("harvest_documents", True), + harvest_datasets=record.harvester_type_specific_configuration.get("harvest_datasets", True), + copy_datasets=record.harvester_type_specific_configuration.get("copy_datasets", False), + copy_documents=record.harvester_type_specific_configuration.get("copy_documents", False), + resource_title_filter=record.harvester_type_specific_configuration.get("resource_title_filter"), + start_date_filter=record.harvester_type_specific_configuration.get("start_date_filter"), + end_date_filter=record.harvester_type_specific_configuration.get("end_date_filter"), + keywords_filter=record.harvester_type_specific_configuration.get("keywords_filter"), + categories_filter=record.harvester_type_specific_configuration.get("categories_filter"), + ) + + +def _check_availability( + http_session, + url: str, + payload_key_to_check: str, + timeout_seconds: typing.Optional[int] = 5, +) -> bool: + try: + response = http_session.get(url, timeout=timeout_seconds) + response.raise_for_status() + except (requests.HTTPError, requests.ConnectionError): + result = False + else: + try: + response_payload = response.json() + except json.JSONDecodeError: + logger.exception("Could not decode server response as valid JSON") + result = False + else: + key_present = response_payload.get(payload_key_to_check) is not None + result = key_present + return result diff --git a/geonode/harvesting/harvesters/wms.py b/geonode/harvesting/harvesters/wms.py new file mode 100644 index 0000000..f4d74ec --- /dev/null +++ b/geonode/harvesting/harvesters/wms.py @@ -0,0 +1,551 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +import typing +import uuid +from datetime import datetime +from functools import lru_cache +from urllib.parse import unquote, urlparse, urlencode, parse_qsl, ParseResult + +import requests +from lxml import etree + +from owslib.map import wms111, wms130 +from owslib.util import clean_ows_url + +from django.conf import settings +from django.contrib.gis import geos +from django.template.defaultfilters import slugify + +from geonode.layers.models import Dataset +from geonode.base.models import ResourceBase +from geonode.layers.enumerations import GXP_PTYPES +from geonode.thumbs.thumbnails import create_thumbnail + +from .. import models +from geonode.utils import ( + XML_PARSER, + get_xpath_value, +) +from .. import resourcedescriptor + +from . import base + +logger = logging.getLogger(__name__) + + +@lru_cache() +def WebMapService( + url, version="1.3.0", xml=None, username=None, password=None, parse_remote_metadata=False, timeout=30, headers=None +): + """ + API for Web Map Service (WMS) methods and metadata. + """ + """wms factory function, returns a version specific WebMapService object + + @type url: string + @param url: url of WFS capabilities document + @type xml: string + @param xml: elementtree object + @type parse_remote_metadata: boolean + @param parse_remote_metadata: whether to fully process MetadataURL elements + @param timeout: time (in seconds) after which requests should timeout + @return: initialized WebFeatureService_2_0_0 object + """ + + clean_url = clean_ows_url(url) + base_ows_url = clean_url + + if version in ["1.1.1"]: + return ( + base_ows_url, + wms111.WebMapService_1_1_1( + clean_url, + version=version, + xml=xml, + parse_remote_metadata=parse_remote_metadata, + username=username, + password=password, + timeout=timeout, + headers=headers, + ), + ) + elif version in ["1.3.0"]: + return ( + base_ows_url, + wms130.WebMapService_1_3_0( + clean_url, + version=version, + xml=xml, + parse_remote_metadata=parse_remote_metadata, + username=username, + password=password, + timeout=timeout, + headers=headers, + ), + ) + raise NotImplementedError( + f"The WMS version ({version}) you requested is not implemented. Please use 1.1.1 or 1.3.0." + ) + + +class OgcWmsHarvester(base.BaseHarvesterWorker): + """Harvester for resources coming from OGC WMS web services""" + + dataset_title_filter: typing.Optional[str] + _base_wms_parameters: typing.Dict = { + "service": "WMS", + "version": "1.3.0", + } + + def __init__(self, *args, dataset_title_filter: typing.Optional[str] = None, **kwargs): + super().__init__(*args, **kwargs) + self.http_session = requests.Session() + self.http_session.headers = {"Content-Type": "application/xml"} + self.dataset_title_filter = dataset_title_filter + + @property + def allows_copying_resources(self) -> bool: + return False + + @classmethod + def from_django_record(cls, record: models.Harvester): + return cls( + record.remote_url, + record.id, + dataset_title_filter=record.harvester_type_specific_configuration.get("dataset_title_filter"), + ) + + @classmethod + def get_extra_config_schema(cls) -> typing.Optional[typing.Dict]: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://geonode.org/harvesting/ogc-wms-harvester.schema.json", + "title": "OGC WMS harvester config", + "description": ( + "A jsonschema for validating configuration option for GeoNode's " "remote OGC WMS harvester" + ), + "type": "object", + "properties": { + "dataset_title_filter": { + "type": "string", + } + }, + "additionalProperties": False, + } + + @classmethod + def get_wms_operations(cls, url, version=None) -> typing.Optional[typing.Dict]: + operations = {} + try: + _url, _parsed_service = WebMapService(url, version=version) + for _op in _parsed_service.operations: + _methods = [] + for _op_method in getattr(_op, "methods", []) if hasattr(_op, "methods") else _op.get("methods", []): + _methods.append({"type": _op_method.get("type", None), "url": _op_method.get("url", None)}) + + _name = getattr(_op, "name", None) if hasattr(_op, "name") else _op.get("name", None) + _formatOptions = ( + getattr(_op, "formatOptions", []) if hasattr(_op, "formatOptions") else _op.get("formatOptions", []) + ) + if _name: + operations[_name] = {"name": _name, "methods": _methods, "formatOptions": _formatOptions} + except Exception as e: + logger.debug(e) + return operations + + def get_ogc_wms_url(self, wms_url, version=None): + ogc_wms_url = f"{wms_url.scheme}://{wms_url.netloc}{wms_url.path}" + try: + operations = OgcWmsHarvester.get_wms_operations(wms_url.geturl(), version=version) + ogc_wms_get_capabilities = operations.get("GetCapabilities", None) + if ogc_wms_get_capabilities and ogc_wms_get_capabilities.get("methods", None): + for _op_method in ogc_wms_get_capabilities.get("methods"): + if _op_method.get("type", None).upper() == "GET" and _op_method.get("url", None): + ogc_wms_url = _op_method.get("url") + break + except Exception as e: + logger.exception(e) + return ogc_wms_url + + def get_capabilities(self) -> requests.Response: + params = self._base_wms_parameters.copy() + params.update( + { + "request": "GetCapabilities", + } + ) + (wms_url, _service, _version, _request) = self._get_cleaned_url_params(self.remote_url) + if _service: + params["service"] = _service + if _version: + params["version"] = _version + if wms_url.query: + for _param in parse_qsl(wms_url.query): + params[_param[0]] = _param[1] + + get_capabilities_response = self.http_session.get( + self.get_ogc_wms_url(wms_url, version=_version), params=params + ) + get_capabilities_response.raise_for_status() + return get_capabilities_response + + def get_num_available_resources(self) -> int: + data = self._get_data() + return len(data["layers"]) + + def list_resources(self, offset: typing.Optional[int] = 0) -> typing.List[base.BriefRemoteResource]: + # look at `tasks.update_harvestable_resources()` in order to understand the purpose of the + # `offset` parameter. Briefly, we try to retrieve resources in batches and we use `offset` to + # control the pagination of the remote service. Unfortunately WMS does not really have the + # concept of pagination and dumps all available layers in a single `GetCapabilities` response. + # With this in mind, we only handle the case where `offset == 0`, which returns all available resources + # and simply return an empty list when `offset != 0` + if offset != 0: + return [] + + resources = [] + data = self._get_data() + for layer in data["layers"]: + name = layer["name"] + title = layer.get("title") or name.rpartition(":")[-1] + resources.append( + base.BriefRemoteResource( + unique_identifier=name, + title=title, + abstract=layer["abstract"], + resource_type="layers", + ) + ) + return resources + + def check_availability(self, timeout_seconds: typing.Optional[int] = 5) -> bool: + try: + response = self.get_capabilities() + except (requests.HTTPError, requests.ConnectionError): + result = False + else: + try: + root = etree.fromstring(response.content, parser=XML_PARSER) + except etree.XMLSyntaxError: + result = False + else: + if "WMS_Capabilities" in root.tag: + result = True + else: + result = False + return result + + def get_geonode_resource_type(self, remote_resource_type: str) -> ResourceBase: + """Return resource type class from resource type string.""" + # WMS just have Layer type on it's resource. + # So whatever remote_resource_type it is, it always return Layer. + return Dataset + + def get_geonode_resource_defaults( + self, + harvested_info: base.HarvestedResourceInfo, + harvestable_resource: models.HarvestableResource, # noqa + ) -> typing.Dict: + defaults = super().get_geonode_resource_defaults(harvested_info, harvestable_resource) + defaults["name"] = harvested_info.resource_descriptor.identification.name + defaults.update(harvested_info.resource_descriptor.additional_parameters) + return defaults + + def get_resource( + self, + harvestable_resource: models.HarvestableResource, + ) -> typing.Optional[base.HarvestedResourceInfo]: + resource_unique_identifier = harvestable_resource.unique_identifier + data = self._get_data() + result = None + try: + relevant_layer = [layer for layer in data["layers"] if layer["name"] == resource_unique_identifier][0] + except IndexError: + logger.exception(f"Could not find resource {resource_unique_identifier!r}") + else: + # WMS does not provide uuid, so needs to generated on the first time + # for update, use uuid from geonode resource + resource_uuid = uuid.uuid4() + if harvestable_resource.geonode_resource: + resource_uuid = uuid.UUID(harvestable_resource.geonode_resource.uuid) + # WMS does not provide the date of the resource. + # Use current time for the date stamp and resource time. + time = datetime.now() + service_name = slugify(self.remote_url)[:255] + contact = resourcedescriptor.RecordDescriptionContact(**data["contact"]) + result = base.HarvestedResourceInfo( + resource_descriptor=resourcedescriptor.RecordDescription( + uuid=resource_uuid, + point_of_contact=contact, + author=contact, + date_stamp=time, + identification=resourcedescriptor.RecordIdentification( + name=relevant_layer["name"], + title=relevant_layer["title"], + date=time, + date_type="", + originator=contact, + place_keywords=[], + other_keywords=relevant_layer["keywords"], + license=[], + abstract=relevant_layer["abstract"], + spatial_extent=relevant_layer["spatial_extent"], + ), + distribution=resourcedescriptor.RecordDistribution( + wms_url=relevant_layer["wms_url"], + ), + reference_systems=[relevant_layer["crs"]], + additional_parameters={ + "alternate": relevant_layer["name"], + "store": service_name, + "workspace": "remoteWorkspace", + "ows_url": relevant_layer["wms_url"], + "ptype": GXP_PTYPES["WMS"], + }, + ), + additional_information=None, + ) + return result + + def _get_data(self) -> typing.Dict: + """Return data from the harvester URL in JSON format.""" + get_capabilities_response = self.get_capabilities() + root = etree.fromstring(get_capabilities_response.content, parser=XML_PARSER) + nsmap = _get_nsmap(root.nsmap) + + layers = [] + leaf_layers = root.xpath("//wms:Layer[not(.//wms:Layer)]", namespaces=nsmap) + for layer_element in leaf_layers: + try: + data = self._layer_element_to_json(layer_element) + title = data["title"] + if self.dataset_title_filter is not None: + if self.dataset_title_filter.lower() not in title.lower(): + continue + layers.append(data) + except Exception as e: + logger.exception(e) + return {"contact": self._get_contact(root), "layers": layers} + + def _get_contact(self, element: etree.Element) -> dict: + """Return contact from element""" + nsmap = _get_nsmap(element.nsmap) + return { + "role": "", + "name": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactPersonPrimary/wms:ContactPerson", nsmap + ), + "organization": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactPersonPrimary/wms:ContactOrganization", nsmap + ), + "position": get_xpath_value(element, "wms:Service/wms:ContactInformation/wms:ContactPosition", nsmap), + "phone_voice": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactVoiceTelephone", nsmap + ), + "phone_facsimile": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactFacsimileTelephone", nsmap + ), + "address_delivery_point": "", + "address_city": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactAddress/wms:City", nsmap + ), + "address_administrative_area": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactAddress/wms:StateOrProvince", nsmap + ), + "address_postal_code": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactAddress/wms:PostCode", nsmap + ), + "address_country": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactAddress/wms:Country", nsmap + ), + "address_email": get_xpath_value( + element, "wms:Service/wms:ContactInformation/wms:ContactElectronicMailAddress", nsmap + ), + } + + def _get_cleaned_url_params(self, url): + # Unquoting URL first so we don't loose existing args + url = unquote(url) + # Extracting url info + parsed_url = urlparse(url) + # Extracting URL arguments from parsed URL + get_args = parsed_url.query + # Converting URL arguments to dict + parsed_get_args = dict(parse_qsl(get_args)) + # Strip out redoundant args + _version = parsed_get_args.pop("version", "1.3.0") if "version" in parsed_get_args else "1.3.0" + _service = parsed_get_args.pop("service") if "service" in parsed_get_args else None + _request = parsed_get_args.pop("request") if "request" in parsed_get_args else None + # Converting URL argument to proper query string + encoded_get_args = urlencode(parsed_get_args, doseq=True) + # Creating new parsed result object based on provided with new + # URL arguments. Same thing happens inside of urlparse. + new_url = ParseResult( + parsed_url.scheme, + parsed_url.netloc, + parsed_url.path, + parsed_url.params, + encoded_get_args, + parsed_url.fragment, + ) + return (new_url, _service, _version, _request) + + def _layer_element_to_json(self, layer_element: etree.Element) -> dict: + """Return json of layer from xml element""" + nsmap = _get_nsmap(layer_element.nsmap) + nsmap["xlink"] = "http://www.w3.org/1999/xlink" + name = get_xpath_value(layer_element, "wms:Name", nsmap) + title = get_xpath_value(layer_element, "wms:Title", nsmap) + abstract = get_xpath_value(layer_element, "wms:Abstract", nsmap) + try: + keywords = layer_element.xpath("wms:KeywordList/wms:Keyword/text()", namespaces=nsmap) + keywords = [str(keyword) for keyword in keywords] + except IndexError: + keywords = [] + + try: + legend_url = layer_element.xpath("wms:Style/wms:LegendURL/wms:OnlineResource", namespaces=nsmap)[0].attrib[ + f"{{{nsmap['xlink']}}}href" + ] + except (IndexError, KeyError): + legend_url = "" + params = {} + (wms_url, _service, _version, _request) = self._get_cleaned_url_params(self.remote_url) + if _service: + params["service"] = _service + if _version: + params["version"] = _version + if wms_url.query: + for _param in parse_qsl(wms_url.query): + params[_param[0]] = _param[1] + + wms_url = self.get_ogc_wms_url(wms_url._replace(query=urlencode(params)), version=_version) + + crs = None + spatial_extent = None + try: + for bbox in layer_element.xpath("wms:BoundingBox", namespaces=nsmap): + crs = bbox.attrib.get("CRS") + if "EPSG:" in crs.upper() or crs.upper() == "CRS:84": + crs = "EPSG:4326" if crs.upper() == "CRS:84" else crs + left_x = bbox.attrib.get("minx") + right_x = bbox.attrib.get("maxx") + lower_y = bbox.attrib.get("miny") + upper_y = bbox.attrib.get("maxy") + + # Preventing if it returns comma as the decimal separator + spatial_extent = geos.Polygon.from_bbox( + ( + float(left_x.replace(",", ".")), + float(lower_y.replace(",", ".")), + float(right_x.replace(",", ".")), + float(upper_y.replace(",", ".")), + ) + ) + break + if not spatial_extent: + crs = None + raise Exception("No suitable wms:BoundingBox element found!") + except Exception as e: + logger.exception(e) + try: + for crs in layer_element.xpath("wms:CRS//text()", namespaces=nsmap): + if "EPSG:" in crs.upper() or crs.upper() == "CRS:84": + crs = "EPSG:4326" if crs.upper() == "CRS:84" else crs + left_x = get_xpath_value( + layer_element, "wms:EX_GeographicBoundingBox/wms:westBoundLongitude", nsmap + ) + right_x = get_xpath_value( + layer_element, "wms:EX_GeographicBoundingBox/wms:eastBoundLongitude", nsmap + ) + lower_y = get_xpath_value( + layer_element, "wms:EX_GeographicBoundingBox/wms:southBoundLatitude", nsmap + ) + upper_y = get_xpath_value( + layer_element, "wms:EX_GeographicBoundingBox/wms:northBoundLatitude", nsmap + ) + + # Preventing if it returns comma as the decimal separator + spatial_extent = geos.Polygon.from_bbox( + ( + float(left_x.replace(",", ".")), + float(lower_y.replace(",", ".")), + float(right_x.replace(",", ".")), + float(upper_y.replace(",", ".")), + ) + ) + break + if not spatial_extent: + crs = None + raise Exception("No suitable wms:CRS element found!") + except Exception as e: + logger.exception(e) + spatial_extent = None + if not spatial_extent: + crs = "EPSG:4326" + spatial_extent = geos.Polygon.from_bbox((-180.0, -90.0, 180.0, 90.0)) + return { + "name": name, + "title": title, + "abstract": abstract, + "crs": crs, + "keywords": keywords, + "spatial_extent": spatial_extent, + "wms_url": wms_url, + "legend_url": legend_url, + } + + def finalize_resource_update( + self, + geonode_resource: ResourceBase, + harvested_info: base.HarvestedResourceInfo, + harvestable_resource: models.HarvestableResource, + ) -> ResourceBase: + """Create a thumbnail with a WMS request.""" + if not geonode_resource.srid: + target_crs = settings.DEFAULT_MAP_CRS + elif "EPSG:" in str(geonode_resource.srid).upper() or "CRS:" in str(geonode_resource.srid).upper(): + target_crs = geonode_resource.srid + else: + target_crs = f"EPSG:{geonode_resource.srid}" + create_thumbnail( + instance=geonode_resource, + # wms_version=harvested_info.resource_descriptor, + bbox=geonode_resource.bbox, + forced_crs=target_crs, + overwrite=True, + ) + + +def _get_nsmap(original: typing.Dict): + """Prepare namespaces dict for running xpath queries. + + lxml complains when a namespaces dict has an entry with None as a key. + + """ + + result = original.copy() + try: + result["wms"] = original[None] + except KeyError: + pass + else: + del result[None] + return result diff --git a/geonode/harvesting/migrations/0001_initial.py b/geonode/harvesting/migrations/0001_initial.py new file mode 100644 index 0000000..94db8e2 --- /dev/null +++ b/geonode/harvesting/migrations/0001_initial.py @@ -0,0 +1,42 @@ +# Generated by Django 2.2.16 on 2021-04-15 13:43 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Harvester', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Harvester name', max_length=100)), + ('remote_url', models.URLField(help_text='Base URL of the remote service that is to be harvested')), + ('scheduling_enabled', models.BooleanField(default=True, help_text='Whether to periodically schedule this harvester to look for resources on the remote service')), + ('update_frequency', models.PositiveIntegerField(default=60, help_text='How often (in minutes) should new harvesting sessions be automatically scheduled? Setting this value to zero has the same effect as setting `scheduling_enabled` to False ')), + ('default_access_permissions', models.JSONField(default=dict, help_text='Default access permissions of harvested resources')), + ('harvester_type', models.CharField(choices=[('geonode.harvesting.harvesters.geonode.GeonodeHarvester', 'geonode.harvesting.harvesters.geonode.GeonodeHarvester')], default='geonode.harvesting.harvesters.geonode.GeonodeHarvester', help_text='Harvester class used to perform harvesting sessions', max_length=255)), + ('harvester_type_specific_configuration', models.JSONField(default=dict, help_text='Configuration specific to each harvester type. Please consult GeoNode documentation on harvesting for more info.')), + ('default_owner', models.ForeignKey(help_text='Default owner of harvested resources', on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='HarvestingSession', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('started', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('ended', models.DateTimeField(blank=True, null=True)), + ('records_harvested', models.IntegerField(default=0)), + ('harvester', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='harvesting.Harvester')), + ], + ), + ] diff --git a/geonode/harvesting/migrations/0002_auto_20210415_1719.py b/geonode/harvesting/migrations/0002_auto_20210415_1719.py new file mode 100644 index 0000000..de01d1d --- /dev/null +++ b/geonode/harvesting/migrations/0002_auto_20210415_1719.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.16 on 2021-04-15 17:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField(choices=[('geonode.harvesting.harvesters.geonode.GeonodeHarvester', 'geonode.harvesting.harvesters.geonode.GeonodeHarvester')], default='geonode.harvesting.harvesters.geonode.GeonodeHarvester', help_text='Harvester class used to perform harvesting sessions. New harvester types can be added by an admin by changing the main GeoNode `settings.py` file', max_length=255), + ), + migrations.AlterField( + model_name='harvestingsession', + name='harvester', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='harvesting_sessions', to='harvesting.Harvester'), + ), + ] diff --git a/geonode/harvesting/migrations/0003_harvestingsession_total_records_found.py b/geonode/harvesting/migrations/0003_harvestingsession_total_records_found.py new file mode 100644 index 0000000..f98c49a --- /dev/null +++ b/geonode/harvesting/migrations/0003_harvestingsession_total_records_found.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-04-16 13:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0002_auto_20210415_1719'), + ] + + operations = [ + migrations.AddField( + model_name='harvestingsession', + name='total_records_found', + field=models.IntegerField(default=0), + ), + ] diff --git a/geonode/harvesting/migrations/0004_auto_20210428_1055.py b/geonode/harvesting/migrations/0004_auto_20210428_1055.py new file mode 100644 index 0000000..d9b1f27 --- /dev/null +++ b/geonode/harvesting/migrations/0004_auto_20210428_1055.py @@ -0,0 +1,38 @@ +# Generated by Django 2.2.16 on 2021-04-28 10:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0003_harvestingsession_total_records_found'), + ] + + operations = [ + migrations.AddField( + model_name='harvester', + name='check_availability_frequency', + field=models.PositiveIntegerField(default=30, help_text='How often (in minutes) should the remote service be checked for availability?'), + ), + migrations.AddField( + model_name='harvester', + name='last_checked_availability', + field=models.DateTimeField(blank=True, help_text='Last time the remote server was checked for availability', null=True), + ), + migrations.AddField( + model_name='harvester', + name='remote_available', + field=models.BooleanField(default=False, help_text='Whether the remote service is known to be available or not'), + ), + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField(choices=[('geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester', 'geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester'), ('geonode.harvesting.harvesters.wms.OgcWmsHarvester', 'geonode.harvesting.harvesters.wms.OgcWmsHarvester')], default='geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester', help_text='Harvester class used to perform harvesting sessions. New harvester types can be added by an admin by changing the main GeoNode `settings.py` file', max_length=255), + ), + migrations.AlterField( + model_name='harvester', + name='harvester_type_specific_configuration', + field=models.JSONField(default=dict, help_text='Configuration specific to each harvester type. Please consult GeoNode documentation on harvesting for more info. This field is mandatory, so at the very least an empty object (i.e. {}) must be supplied.'), + ), + ] diff --git a/geonode/harvesting/migrations/0005_auto_20210428_1100.py b/geonode/harvesting/migrations/0005_auto_20210428_1100.py new file mode 100644 index 0000000..4e33c23 --- /dev/null +++ b/geonode/harvesting/migrations/0005_auto_20210428_1100.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-04-28 11:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0004_auto_20210428_1055'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='remote_available', + field=models.BooleanField(default=False, editable=False, help_text='Whether the remote service is known to be available or not'), + ), + ] diff --git a/geonode/harvesting/migrations/0006_harvester_availability_check_task.py b/geonode/harvesting/migrations/0006_harvester_availability_check_task.py new file mode 100644 index 0000000..017dd28 --- /dev/null +++ b/geonode/harvesting/migrations/0006_harvester_availability_check_task.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.16 on 2021-04-28 11:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0005_auto_20210428_1100'), + ] + + operations = [ + ] diff --git a/geonode/harvesting/migrations/0007_auto_20210519_1034.py b/geonode/harvesting/migrations/0007_auto_20210519_1034.py new file mode 100644 index 0000000..6638263 --- /dev/null +++ b/geonode/harvesting/migrations/0007_auto_20210519_1034.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.16 on 2021-05-19 10:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0006_harvester_availability_check_task'), + ] + + operations = [ + migrations.AddField( + model_name='harvester', + name='delete_orphan_resources_automatically', + field=models.BooleanField(default=False, help_text='Orphan resources are those that have previously been created by means of a harvesting operation but that GeoNode can no longer find on the remote service being harvested. Should these resources be deleted from GeoNode automatically? This also applies to when a harvester configuration is deleted, in which case all of the resources that originated from that harvester are now considered to be orphan.'), + ), + migrations.AddField( + model_name='harvester', + name='harvest_new_resources_by_default', + field=models.BooleanField(default=False, help_text='Should new resources be harvested automatically without explicit selection?'), + ), + migrations.AddField( + model_name='harvester', + name='last_updated', + field=models.DateTimeField(auto_now=True, help_text='Date of last update to the harvester configuration.'), + ), + ] diff --git a/geonode/harvesting/migrations/0008_auto_20210519_1055.py b/geonode/harvesting/migrations/0008_auto_20210519_1055.py new file mode 100644 index 0000000..1bf9d36 --- /dev/null +++ b/geonode/harvesting/migrations/0008_auto_20210519_1055.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.16 on 2021-05-19 10:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0058_thesaurus_order'), + ('harvesting', '0007_auto_20210519_1034'), + ] + + operations = [ + migrations.CreateModel( + name='HarvestableResource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('unique_identifier', models.CharField(help_text='Identifier that allows referencing the resource on its remote service in a unique fashion. This is usually automatically filled by the harvester worker. The harvester worker needs to know how to either read or generate this from each remote resource in order to be able to compare the availability of resources between consecutive harvesting sessions.', max_length=255)), + ('title', models.CharField(max_length=255)), + ('geonode_resource', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='base.ResourceBase')), + ('harvester', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='harvesting.Harvester')), + ], + ), + migrations.AddConstraint( + model_name='harvestableresource', + constraint=models.UniqueConstraint(fields=('harvester', 'unique_identifier'), name='unique_id_for_harvester'), + ), + ] diff --git a/geonode/harvesting/migrations/0009_auto_20210519_1638.py b/geonode/harvesting/migrations/0009_auto_20210519_1638.py new file mode 100644 index 0000000..c149dbd --- /dev/null +++ b/geonode/harvesting/migrations/0009_auto_20210519_1638.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.16 on 2021-05-19 16:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0008_auto_20210519_1055'), + ] + + operations = [ + migrations.AlterField( + model_name='harvestableresource', + name='harvester', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='harvestable_resources', to='harvesting.Harvester'), + ), + ] diff --git a/geonode/harvesting/migrations/0010_harvestableresource_should_be_harvested.py b/geonode/harvesting/migrations/0010_harvestableresource_should_be_harvested.py new file mode 100644 index 0000000..902c40d --- /dev/null +++ b/geonode/harvesting/migrations/0010_harvestableresource_should_be_harvested.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-05-20 17:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0009_auto_20210519_1638'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='should_be_harvested', + field=models.BooleanField(default=False), + ), + ] diff --git a/geonode/harvesting/migrations/0011_harvester_status.py b/geonode/harvesting/migrations/0011_harvester_status.py new file mode 100644 index 0000000..83d9802 --- /dev/null +++ b/geonode/harvesting/migrations/0011_harvester_status.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-05-20 20:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0010_harvestableresource_should_be_harvested'), + ] + + operations = [ + migrations.AddField( + model_name='harvester', + name='status', + field=models.CharField(choices=[('ready', 'ready'), ('updating', 'updating'), ('harvesting', 'harvesting')], default='ready', max_length=20), + ), + ] diff --git a/geonode/harvesting/migrations/0012_auto_20210520_2055.py b/geonode/harvesting/migrations/0012_auto_20210520_2055.py new file mode 100644 index 0000000..b5aee38 --- /dev/null +++ b/geonode/harvesting/migrations/0012_auto_20210520_2055.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.16 on 2021-05-20 20:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0011_harvester_status'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='available', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='harvestableresource', + name='last_updated', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/geonode/harvesting/migrations/0013_auto_20210520_2159.py b/geonode/harvesting/migrations/0013_auto_20210520_2159.py new file mode 100644 index 0000000..60da609 --- /dev/null +++ b/geonode/harvesting/migrations/0013_auto_20210520_2159.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.16 on 2021-05-20 21:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0012_auto_20210520_2055'), + ] + + operations = [ + migrations.AddField( + model_name='harvester', + name='last_check_harvestable_resources_message', + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name='harvester', + name='last_checked_harvestable_resources', + field=models.DateTimeField(blank=True, help_text='Last time the remote server was checked for harvestable resources', null=True), + ), + ] diff --git a/geonode/harvesting/migrations/0014_auto_20210526_1521.py b/geonode/harvesting/migrations/0014_auto_20210526_1521.py new file mode 100644 index 0000000..12d1b10 --- /dev/null +++ b/geonode/harvesting/migrations/0014_auto_20210526_1521.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-05-26 15:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0013_auto_20210520_2159'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='status', + field=models.CharField(choices=[('ready', 'ready'), ('updating-harvestable-resources', 'updating-harvestable-resources'), ('harvesting-resources', 'harvesting-resources'), ('checking-availability', 'checking-availability')], default='ready', max_length=50), + ), + ] diff --git a/geonode/harvesting/migrations/0015_harvestableresource_status.py b/geonode/harvesting/migrations/0015_harvestableresource_status.py new file mode 100644 index 0000000..216b1e0 --- /dev/null +++ b/geonode/harvesting/migrations/0015_harvestableresource_status.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-05-26 15:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0014_auto_20210526_1521'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='status', + field=models.CharField(choices=[('ready', 'ready'), ('updating-harvestable-resource', 'updating-harvestable-resource'), ('harvesting-resource', 'harvesting-resource')], default='ready', max_length=50), + ), + ] diff --git a/geonode/harvesting/migrations/0016_harvestableresource_remote_resource_type.py b/geonode/harvesting/migrations/0016_harvestableresource_remote_resource_type.py new file mode 100644 index 0000000..e4a9c61 --- /dev/null +++ b/geonode/harvesting/migrations/0016_harvestableresource_remote_resource_type.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-06-04 00:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0015_harvestableresource_status'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='remote_resource_type', + field=models.CharField(blank=True, help_text='Type of the resource in the remote service. Each harvester worker knows how to fill this field, in accordance with the resources for which harvesting is supported', max_length=255), + ), + ] diff --git a/geonode/harvesting/migrations/0017_auto_20210614_1530.py b/geonode/harvesting/migrations/0017_auto_20210614_1530.py new file mode 100644 index 0000000..49e3456 --- /dev/null +++ b/geonode/harvesting/migrations/0017_auto_20210614_1530.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2 on 2021-06-14 15:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0016_harvestableresource_remote_resource_type'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='default_access_permissions', + field=models.JSONField(help_text='Default access permissions of harvested resources'), + ), + migrations.AlterField( + model_name='harvester', + name='harvester_type_specific_configuration', + field=models.JSONField(help_text='Configuration specific to each harvester type. Please consult GeoNode documentation on harvesting for more info. This field is mandatory, so at the very least an empty object (i.e. {}) must be supplied.'), + ), + ] diff --git a/geonode/harvesting/migrations/0018_auto_20210614_1733.py b/geonode/harvesting/migrations/0018_auto_20210614_1733.py new file mode 100644 index 0000000..31a2d51 --- /dev/null +++ b/geonode/harvesting/migrations/0018_auto_20210614_1733.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2 on 2021-06-14 17:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0017_auto_20210614_1530'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='default_access_permissions', + field=models.JSONField(default=dict, help_text='Default access permissions of harvested resources'), + ), + migrations.AlterField( + model_name='harvester', + name='harvester_type_specific_configuration', + field=models.JSONField(default=dict, help_text='Configuration specific to each harvester type. Please consult GeoNode documentation on harvesting for more info. This field is mandatory, so at the very least an empty object (i.e. {}) must be supplied.'), + ), + ] diff --git a/geonode/harvesting/migrations/0019_auto_20210614_1735.py b/geonode/harvesting/migrations/0019_auto_20210614_1735.py new file mode 100644 index 0000000..ce86e28 --- /dev/null +++ b/geonode/harvesting/migrations/0019_auto_20210614_1735.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2 on 2021-06-14 17:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0018_auto_20210614_1733'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='default_access_permissions', + field=models.JSONField(blank=True, default=dict, help_text='Default access permissions of harvested resources'), + ), + migrations.AlterField( + model_name='harvester', + name='harvester_type_specific_configuration', + field=models.JSONField(blank=True, default=dict, help_text='Configuration specific to each harvester type. Please consult GeoNode documentation on harvesting for more info. This field is mandatory, so at the very least an empty object (i.e. {}) must be supplied.'), + ), + ] diff --git a/geonode/harvesting/migrations/0020_alter_harvester_default_access_permissions.py b/geonode/harvesting/migrations/0020_alter_harvester_default_access_permissions.py new file mode 100644 index 0000000..194e4af --- /dev/null +++ b/geonode/harvesting/migrations/0020_alter_harvester_default_access_permissions.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-06-23 17:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0019_auto_20210614_1735'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='default_access_permissions', + field=models.JSONField(default=dict, help_text='Default access permissions of harvested resources'), + ), + ] diff --git a/geonode/harvesting/migrations/0021_alter_harvester_default_access_permissions.py b/geonode/harvesting/migrations/0021_alter_harvester_default_access_permissions.py new file mode 100644 index 0000000..4a23669 --- /dev/null +++ b/geonode/harvesting/migrations/0021_alter_harvester_default_access_permissions.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-06-23 17:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0020_alter_harvester_default_access_permissions'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='default_access_permissions', + field=models.JSONField(blank=True, default=dict, help_text='Default access permissions of harvested resources'), + ), + ] diff --git a/geonode/harvesting/migrations/0022_alter_harvester_default_access_permissions.py b/geonode/harvesting/migrations/0022_alter_harvester_default_access_permissions.py new file mode 100644 index 0000000..235ccb7 --- /dev/null +++ b/geonode/harvesting/migrations/0022_alter_harvester_default_access_permissions.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-06-23 17:19 + +from django.db import migrations, models +import geonode.harvesting.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0021_alter_harvester_default_access_permissions'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='default_access_permissions', + field=models.JSONField(help_text='Default access permissions of harvested resources'), + ), + ] diff --git a/geonode/harvesting/migrations/0023_harvestableresource_last_refreshed.py b/geonode/harvesting/migrations/0023_harvestableresource_last_refreshed.py new file mode 100644 index 0000000..9130ffb --- /dev/null +++ b/geonode/harvesting/migrations/0023_harvestableresource_last_refreshed.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.4 on 2021-06-24 00:05 + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0022_alter_harvester_default_access_permissions'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='last_refreshed', + field=models.DateTimeField(default=datetime.datetime(2021, 6, 24, 0, 5, 26, 321641, tzinfo=utc)), + preserve_default=False, + ), + ] diff --git a/geonode/harvesting/migrations/0024_remove_harvestableresource_available.py b/geonode/harvesting/migrations/0024_remove_harvestableresource_available.py new file mode 100644 index 0000000..78cf654 --- /dev/null +++ b/geonode/harvesting/migrations/0024_remove_harvestableresource_available.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-06-24 13:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0023_harvestableresource_last_refreshed'), + ] + + operations = [ + migrations.RemoveField( + model_name='harvestableresource', + name='available', + ), + ] diff --git a/geonode/harvesting/migrations/0025_auto_20210624_1459.py b/geonode/harvesting/migrations/0025_auto_20210624_1459.py new file mode 100644 index 0000000..fb2bd2b --- /dev/null +++ b/geonode/harvesting/migrations/0025_auto_20210624_1459.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.4 on 2021-06-24 14:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0024_remove_harvestableresource_available'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='last_harvested', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='harvestableresource', + name='last_harvesting_message', + field=models.TextField(blank=True), + ), + ] diff --git a/geonode/harvesting/migrations/0026_harvestableresource_last_harvesting_succeeded.py b/geonode/harvesting/migrations/0026_harvestableresource_last_harvesting_succeeded.py new file mode 100644 index 0000000..a1a31bf --- /dev/null +++ b/geonode/harvesting/migrations/0026_harvestableresource_last_harvesting_succeeded.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-06-24 15:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0025_auto_20210624_1459'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='last_harvesting_succeeded', + field=models.BooleanField(default=False), + ), + ] diff --git a/geonode/harvesting/migrations/0027_remove_harvester_default_access_permissions.py b/geonode/harvesting/migrations/0027_remove_harvester_default_access_permissions.py new file mode 100644 index 0000000..3ca9ea8 --- /dev/null +++ b/geonode/harvesting/migrations/0027_remove_harvester_default_access_permissions.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.4 on 2021-07-08 13:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ('harvesting', '0026_harvestableresource_last_harvesting_succeeded'), + ] + + operations = [ + migrations.RemoveField( + model_name='harvester', + name='default_access_permissions', + ), + ] diff --git a/geonode/harvesting/migrations/0028_harvester_num_harvestable_resources.py b/geonode/harvesting/migrations/0028_harvester_num_harvestable_resources.py new file mode 100644 index 0000000..0e65387 --- /dev/null +++ b/geonode/harvesting/migrations/0028_harvester_num_harvestable_resources.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-08-12 04:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0027_remove_harvester_default_access_permissions'), + ] + + operations = [ + migrations.AddField( + model_name='harvester', + name='num_harvestable_resources', + field=models.IntegerField(default=0), + ), + ] diff --git a/geonode/harvesting/migrations/0029_dynamic_harvester_type_choices.py b/geonode/harvesting/migrations/0029_dynamic_harvester_type_choices.py new file mode 100644 index 0000000..7fb877a --- /dev/null +++ b/geonode/harvesting/migrations/0029_dynamic_harvester_type_choices.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.4 on 2021-06-28 18:26 +# Hand edited in order to set choices for `modelsHarvester.harvester_type` to come from settings in a dynamic fashion +# This shall prevent Django autogenerating new migration files for `geonode.harvesting` +# whenever new custom harvester classes are added to the settings + +from django.db import migrations, models + +from .. import config + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0028_harvester_num_harvestable_resources'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField( + choices=[(value, value) for value in config.get_setting("HARVESTER_CLASSES")], + default='geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester', + help_text=( + 'Harvester class used to perform harvesting sessions. New harvester types can be added by an admin by changing the ' + 'main GeoNode `settings.py` file' + ), + max_length=255 + ), + ), + ] diff --git a/geonode/harvesting/migrations/0030_auto_20210917_1125.py b/geonode/harvesting/migrations/0030_auto_20210917_1125.py new file mode 100644 index 0000000..29e535b --- /dev/null +++ b/geonode/harvesting/migrations/0030_auto_20210917_1125.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.4 on 2021-09-17 11:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0029_dynamic_harvester_type_choices'), + ] + + operations = [ + migrations.AddField( + model_name='harvestingsession', + name='records_to_harvest', + field=models.IntegerField(default=0, editable=False), + ), + migrations.AddField( + model_name='harvestingsession', + name='status', + field=models.CharField(choices=[('on-going', 'on-going'), ('finished-all-ok', 'finished-all-ok'), ('finished-all-failed', 'finished-all-failed'), ('finished-some-failed', 'finished-some-failed')], default='on-going', editable=False, max_length=50), + ), + migrations.AlterField( + model_name='harvester', + name='num_harvestable_resources', + field=models.IntegerField(blank=True, default=0), + ), + ] diff --git a/geonode/harvesting/migrations/0031_remove_harvestingsession_total_records_found.py b/geonode/harvesting/migrations/0031_remove_harvestingsession_total_records_found.py new file mode 100644 index 0000000..2bee861 --- /dev/null +++ b/geonode/harvesting/migrations/0031_remove_harvestingsession_total_records_found.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-09-17 11:27 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0030_auto_20210917_1125'), + ] + + operations = [ + migrations.RemoveField( + model_name='harvestingsession', + name='total_records_found', + ), + ] diff --git a/geonode/harvesting/migrations/0032_harvestingsession_session_details.py b/geonode/harvesting/migrations/0032_harvestingsession_session_details.py new file mode 100644 index 0000000..a24ab8c --- /dev/null +++ b/geonode/harvesting/migrations/0032_harvestingsession_session_details.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-09-17 11:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0031_remove_harvestingsession_total_records_found'), + ] + + operations = [ + migrations.AddField( + model_name='harvestingsession', + name='session_details', + field=models.TextField(blank=True, help_text='Details about the harvesting session'), + ), + ] diff --git a/geonode/harvesting/migrations/0033_alter_harvestingsession_status.py b/geonode/harvesting/migrations/0033_alter_harvestingsession_status.py new file mode 100644 index 0000000..5911572 --- /dev/null +++ b/geonode/harvesting/migrations/0033_alter_harvestingsession_status.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-09-17 15:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0032_harvestingsession_session_details'), + ] + + operations = [ + migrations.AlterField( + model_name='harvestingsession', + name='status', + field=models.CharField(choices=[('pending', 'pending'), ('on-going', 'on-going'), ('finished-all-ok', 'finished-all-ok'), ('finished-all-failed', 'finished-all-failed'), ('finished-some-failed', 'finished-some-failed')], default='pending', editable=False, max_length=50), + ), + ] diff --git a/geonode/harvesting/migrations/0034_harvester_unique name.py b/geonode/harvesting/migrations/0034_harvester_unique name.py new file mode 100644 index 0000000..8824e1e --- /dev/null +++ b/geonode/harvesting/migrations/0034_harvester_unique name.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-09-17 16:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0033_alter_harvestingsession_status'), + ] + + operations = [ + migrations.AddConstraint( + model_name='harvester', + constraint=models.UniqueConstraint(fields=('name',), name='unique name'), + ), + ] diff --git a/geonode/harvesting/migrations/0035_replace_geonode_harvester_module_name.py b/geonode/harvesting/migrations/0035_replace_geonode_harvester_module_name.py new file mode 100644 index 0000000..96a9072 --- /dev/null +++ b/geonode/harvesting/migrations/0035_replace_geonode_harvester_module_name.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.4 on 2021-09-22 10:27 + +from django.db import migrations + + +def update_geonode_harvester_module_name(apps, schema_editor): + harvester_model = apps.get_model("harvesting", "Harvester") + old_module_name = "geonode" + new_module_name = "geonodeharvester" + fragment_pattern = "geonode.harvesting.harvesters.{module}." + old_path_fragment = fragment_pattern.format(module=old_module_name) + new_path_fragment = fragment_pattern.format(module=new_module_name) + for harvester in harvester_model.objects.filter(harvester_type__startswith=old_path_fragment): + harvester.harvester_type = harvester.harvester_type.replace(old_path_fragment, new_path_fragment) + harvester.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0034_harvester_unique name'), + ] + + operations = [ + migrations.RunPython(update_geonode_harvester_module_name) + ] diff --git a/geonode/harvesting/migrations/0036_alter_harvester_harvester_type.py b/geonode/harvesting/migrations/0036_alter_harvester_harvester_type.py new file mode 100644 index 0000000..eeb972b --- /dev/null +++ b/geonode/harvesting/migrations/0036_alter_harvester_harvester_type.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.4 on 2021-09-22 13:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0035_replace_geonode_harvester_module_name'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField( + choices=[ + ( + 'geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester', + 'geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester' + ), + ( + 'geonode.harvesting.harvesters.wms.OgcWmsHarvester', + 'geonode.harvesting.harvesters.wms.OgcWmsHarvester' + ), + ], + default='geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester', + help_text=( + 'Harvester class used to perform harvesting sessions. New harvester types can be ' + 'added by an admin by changing the main GeoNode `settings.py` file' + ), + max_length=255 + ), + ), + ] diff --git a/geonode/harvesting/migrations/0037_alter_harvester_name.py b/geonode/harvesting/migrations/0037_alter_harvester_name.py new file mode 100644 index 0000000..89c1ab5 --- /dev/null +++ b/geonode/harvesting/migrations/0037_alter_harvester_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.7 on 2021-09-30 17:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0036_alter_harvester_harvester_type'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='name', + field=models.CharField(help_text='Harvester name', max_length=255), + ), + ] diff --git a/geonode/harvesting/migrations/0038_alter_harvester_status.py b/geonode/harvesting/migrations/0038_alter_harvester_status.py new file mode 100644 index 0000000..45c87fb --- /dev/null +++ b/geonode/harvesting/migrations/0038_alter_harvester_status.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.4 on 2021-09-24 09:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0037_alter_harvester_name'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='status', + field=models.CharField( + choices=[ + ('ready', 'ready'), + ('updating-harvestable-resources', 'updating-harvestable-resources'), + ('aborting-update-harvestable-resources', 'aborting-update-harvestable-resources'), + ('harvesting-resources', 'harvesting-resources'), + ('aborting-harvesting-resources', 'aborting-harvesting-resources'), + ('checking-availability', 'checking-availability') + ], + default='ready', + max_length=50 + ), + ), + ] diff --git a/geonode/harvesting/migrations/0039_harvestableresource_abstract.py b/geonode/harvesting/migrations/0039_harvestableresource_abstract.py new file mode 100644 index 0000000..b7edd33 --- /dev/null +++ b/geonode/harvesting/migrations/0039_harvestableresource_abstract.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.7 on 2021-10-05 10:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0038_alter_harvester_status'), + ] + + operations = [ + migrations.AddField( + model_name='harvestableresource', + name='abstract', + field=models.TextField(blank=True, max_length=2000), + ), + ] diff --git a/geonode/harvesting/migrations/0040_auto_20210927_1455.py b/geonode/harvesting/migrations/0040_auto_20210927_1455.py new file mode 100644 index 0000000..f3e567d --- /dev/null +++ b/geonode/harvesting/migrations/0040_auto_20210927_1455.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.4 on 2021-09-27 14:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0039_harvestableresource_abstract'), + ] + + operations = [ + migrations.AlterField( + model_name='harvestingsession', + name='status', + field=models.CharField(choices=[('pending', 'pending'), ('on-going', 'on-going'), ('finished-all-ok', 'finished-all-ok'), ('finished-all-failed', 'finished-all-failed'), ('finished-some-failed', 'finished-some-failed'), ('aborting', 'aborting'), ('aborted', 'aborted')], default='pending', editable=False, max_length=50), + ), + migrations.CreateModel( + name='AsynchronousHarvestingSession', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('session_type', models.CharField(choices=[('harvesting', 'harvesting'), ('discover-harvestable-resources', 'discover-harvestable-resources')], editable=False, max_length=50)), + ('started', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('ended', models.DateTimeField(blank=True, null=True)), + ('status', models.CharField(choices=[('pending', 'pending'), ('on-going', 'on-going'), ('finished-all-ok', 'finished-all-ok'), ('finished-all-failed', 'finished-all-failed'), ('finished-some-failed', 'finished-some-failed'), ('aborting', 'aborting'), ('aborted', 'aborted')], default='pending', editable=False, max_length=50)), + ('details', models.TextField(blank=True)), + ('total_records_to_process', models.IntegerField(default=0, editable=False, help_text='Number of records being processed in this session')), + ('records_done', models.IntegerField(default=0, help_text='Number of records that have already been processed')), + ('harvester', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sessions', to='harvesting.harvester')), + ], + ), + ] diff --git a/geonode/harvesting/migrations/0041_delete_harvestingsession.py b/geonode/harvesting/migrations/0041_delete_harvestingsession.py new file mode 100644 index 0000000..553ef4c --- /dev/null +++ b/geonode/harvesting/migrations/0041_delete_harvestingsession.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.4 on 2021-09-27 15:01 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0040_auto_20210927_1455'), + ] + + operations = [ + migrations.DeleteModel( + name='HarvestingSession', + ), + ] diff --git a/geonode/harvesting/migrations/0042_auto_20210930_1155.py b/geonode/harvesting/migrations/0042_auto_20210930_1155.py new file mode 100644 index 0000000..30a60fd --- /dev/null +++ b/geonode/harvesting/migrations/0042_auto_20210930_1155.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.4 on 2021-09-30 11:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0041_delete_harvestingsession'), + ] + + operations = [ + migrations.AddField( + model_name='harvester', + name='refresh_harvestable_resources_update_frequency', + field=models.PositiveIntegerField(default=30, help_text='How often (in minutes) should new refresh sessions be automatically scheduled?'), + ), + migrations.AlterField( + model_name='harvester', + name='check_availability_frequency', + field=models.PositiveIntegerField(default=10, help_text='How often (in minutes) should the remote service be checked for availability?'), + ), + migrations.AlterField( + model_name='harvester', + name='update_frequency', + field=models.PositiveIntegerField(default=60, help_text='How often (in minutes) should new harvesting sessions be automatically scheduled?'), + ), + ] diff --git a/geonode/harvesting/migrations/0043_harvester_harvesting_session_update_frequency.py b/geonode/harvesting/migrations/0043_harvester_harvesting_session_update_frequency.py new file mode 100644 index 0000000..e3f7f3f --- /dev/null +++ b/geonode/harvesting/migrations/0043_harvester_harvesting_session_update_frequency.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-09-30 11:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0042_auto_20210930_1155'), + ] + + operations = [ + migrations.AddField( + model_name='harvester', + name='harvesting_session_update_frequency', + field=models.PositiveIntegerField(default=60, help_text='How often (in minutes) should new harvesting sessions be automatically scheduled?'), + ), + ] diff --git a/geonode/harvesting/migrations/0044_migrate_harvester_harvesting_session_update_frequency.py b/geonode/harvesting/migrations/0044_migrate_harvester_harvesting_session_update_frequency.py new file mode 100644 index 0000000..9bdd6d0 --- /dev/null +++ b/geonode/harvesting/migrations/0044_migrate_harvester_harvesting_session_update_frequency.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.4 on 2021-09-30 12:01 + +from django.db import migrations + + +def migrate_harvesting_session_update_frequency(apps, schema_editor): + harvester_model = apps.get_model("harvesting", "Harvester") + for harvester in harvester_model.objects.all(): + harvester.harvesting_session_update_frequency = harvester.update_frequency + harvester.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0043_harvester_harvesting_session_update_frequency'), + ] + + operations = [ + migrations.RunPython(migrate_harvesting_session_update_frequency, reverse_code=migrations.RunPython.noop), + ] diff --git a/geonode/harvesting/migrations/0045_remove_harvester_update_frequency.py b/geonode/harvesting/migrations/0045_remove_harvester_update_frequency.py new file mode 100644 index 0000000..02e2824 --- /dev/null +++ b/geonode/harvesting/migrations/0045_remove_harvester_update_frequency.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-09-30 12:12 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0044_migrate_harvester_harvesting_session_update_frequency'), + ] + + operations = [ + migrations.RemoveField( + model_name='harvester', + name='update_frequency', + ), + ] diff --git a/geonode/harvesting/migrations/0046_alter_harvester_harvester_type.py b/geonode/harvesting/migrations/0046_alter_harvester_harvester_type.py new file mode 100644 index 0000000..61abf01 --- /dev/null +++ b/geonode/harvesting/migrations/0046_alter_harvester_harvester_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.7 on 2021-10-08 22:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0045_remove_harvester_update_frequency'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField(choices=[('geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', 'geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker'), ('geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester', 'geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester'), ('geonode.harvesting.harvesters.geonodeharvester.GeonodeCurrentHarvester', 'geonode.harvesting.harvesters.geonodeharvester.GeonodeCurrentHarvester'), ('geonode.harvesting.harvesters.wms.OgcWmsHarvester', 'geonode.harvesting.harvesters.wms.OgcWmsHarvester')], default='geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', help_text='Harvester class used to perform harvesting sessions. New harvester types can be added by an admin by changing the main GeoNode `settings.py` file', max_length=255), + ), + ] diff --git a/geonode/harvesting/migrations/0047_convert_geonode_harvesters_to_unified.py b/geonode/harvesting/migrations/0047_convert_geonode_harvesters_to_unified.py new file mode 100644 index 0000000..c15a94d --- /dev/null +++ b/geonode/harvesting/migrations/0047_convert_geonode_harvesters_to_unified.py @@ -0,0 +1,37 @@ +# Generated by Django 3.2.7 on 2021-10-11 11:51 + +from django.db import migrations + +_INDIVIDUAL_HARVESTER_PATHS = [ + "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester", + "geonode.harvesting.harvesters.geonodeharvester.GeonodeCurrentHarvester", +] +_UNIFIED_HARVESTER_PATH = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" + + +def convert_geonode_harvesters_to_unified(apps, schema_editor): + """Convert GeoNode harvesters to use the unified harvester""" + harvester_model = apps.get_model('harvesting', 'Harvester') + for harvester in harvester_model.objects.all(): + if harvester.harvester_type in _INDIVIDUAL_HARVESTER_PATHS: + harvester.harvester_type = _UNIFIED_HARVESTER_PATH + harvester.save() + + +def reverse_harvester_conversion(apps, schema_editor): + harvester_model = apps.get_model('harvesting', 'Harvester') + for harvester in harvester_model.objects.all(): + if harvester.harvester_type == _UNIFIED_HARVESTER_PATH: + harvester.harvester_type = _INDIVIDUAL_HARVESTER_PATHS[0] + harvester.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0046_alter_harvester_harvester_type'), + ] + + operations = [ + migrations.RunPython(convert_geonode_harvesters_to_unified, reverse_code=reverse_harvester_conversion), + ] diff --git a/geonode/harvesting/migrations/0048_alter_harvester_harvester_type.py b/geonode/harvesting/migrations/0048_alter_harvester_harvester_type.py new file mode 100644 index 0000000..f891cff --- /dev/null +++ b/geonode/harvesting/migrations/0048_alter_harvester_harvester_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.7 on 2021-10-11 12:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0047_convert_geonode_harvesters_to_unified'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField(choices=[('geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', 'geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker'), ('geonode.harvesting.harvesters.wms.OgcWmsHarvester', 'geonode.harvesting.harvesters.wms.OgcWmsHarvester')], default='geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', help_text='Harvester class used to perform harvesting sessions. New harvester types can be added by an admin by changing the main GeoNode `settings.py` file', max_length=255), + ), + ] diff --git a/geonode/harvesting/migrations/0049_alter_harvester_harvester_type.py b/geonode/harvesting/migrations/0049_alter_harvester_harvester_type.py new file mode 100644 index 0000000..bab927d --- /dev/null +++ b/geonode/harvesting/migrations/0049_alter_harvester_harvester_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-10-12 13:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0048_alter_harvester_harvester_type'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField(choices=[('geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', 'geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker'), ('geonode.harvesting.harvesters.wms.OgcWmsHarvester', 'geonode.harvesting.harvesters.wms.OgcWmsHarvester'), ('geonode.harvesting.harvesters.arcgis.ArcgisHarvesterWorker', 'geonode.harvesting.harvesters.arcgis.ArcgisHarvesterWorker')], default='geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', help_text='Harvester class used to perform harvesting sessions. New harvester types can be added by an admin by changing the main GeoNode `settings.py` file', max_length=255), + ), + ] diff --git a/geonode/harvesting/migrations/__init__.py b/geonode/harvesting/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/harvesting/models.py b/geonode/harvesting/models.py new file mode 100644 index 0000000..977f69b --- /dev/null +++ b/geonode/harvesting/models.py @@ -0,0 +1,456 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import datetime as dt +import logging +import typing + +import jsonschema +import jsonschema.exceptions +from django.conf import settings +from django.contrib import admin +from django.core.exceptions import ValidationError +from django.db import models +from django.utils import timezone +from django.utils.module_loading import import_string +from django.utils.translation import gettext_lazy as _ + +from geonode import celery_app + +from .config import get_setting + +logger = logging.getLogger(__name__) + + +class Harvester(models.Model): + STATUS_READY = "ready" + STATUS_UPDATING_HARVESTABLE_RESOURCES = "updating-harvestable-resources" + STATUS_ABORTING_UPDATE_HARVESTABLE_RESOURCES = "aborting-update-harvestable-resources" + STATUS_PERFORMING_HARVESTING = "harvesting-resources" + STATUS_ABORTING_PERFORMING_HARVESTING = "aborting-harvesting-resources" + STATUS_CHECKING_AVAILABILITY = "checking-availability" + STATUS_CHOICES = [ + (STATUS_READY, _("ready")), + (STATUS_UPDATING_HARVESTABLE_RESOURCES, _("updating-harvestable-resources")), + (STATUS_ABORTING_UPDATE_HARVESTABLE_RESOURCES, _("aborting-update-harvestable-resources")), + (STATUS_PERFORMING_HARVESTING, _("harvesting-resources")), + (STATUS_ABORTING_PERFORMING_HARVESTING, _("aborting-harvesting-resources")), + (STATUS_CHECKING_AVAILABILITY, _("checking-availability")), + ] + + name = models.CharField(max_length=255, help_text=_("Harvester name")) + status = models.CharField(max_length=50, choices=STATUS_CHOICES, default=STATUS_READY) + remote_url = models.URLField(help_text=_("Base URL of the remote service that is to be harvested")) + scheduling_enabled = models.BooleanField( + help_text=_("Whether to periodically schedule this harvester to look for resources on " "the remote service"), + default=True, + ) + harvesting_session_update_frequency = models.PositiveIntegerField( + help_text=_("How often (in minutes) should new harvesting sessions be automatically scheduled?"), default=60 + ) + refresh_harvestable_resources_update_frequency = models.PositiveIntegerField( + help_text=_("How often (in minutes) should new refresh sessions be automatically scheduled?"), default=30 + ) + remote_available = models.BooleanField( + help_text=_("Whether the remote service is known to be available or not"), editable=False, default=False + ) + check_availability_frequency = models.PositiveIntegerField( + help_text=_("How often (in minutes) should the remote service be checked for " "availability?"), default=10 + ) + last_checked_availability = models.DateTimeField( + help_text=_("Last time the remote server was checked for availability"), blank=True, null=True + ) + last_checked_harvestable_resources = models.DateTimeField( + help_text=_("Last time the remote server was checked for harvestable resources"), + blank=True, + null=True, + ) + last_check_harvestable_resources_message = models.TextField(blank=True) + default_owner = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, help_text=_("Default owner of harvested resources") + ) + harvest_new_resources_by_default = models.BooleanField( + help_text=_("Should new resources be harvested automatically without " "explicit selection?"), + default=False, + ) + delete_orphan_resources_automatically = models.BooleanField( + help_text=_( + "Orphan resources are those that have previously been created by means of " + "a harvesting operation but that GeoNode can no longer find on the remote " + "service being harvested. Should these resources be deleted from GeoNode " + "automatically? This also applies to when a harvester configuration is " + "deleted, in which case all of the resources that originated from that " + "harvester are now considered to be orphan." + ), + default=False, + ) + last_updated = models.DateTimeField( + help_text=_("Date of last update to the harvester configuration."), auto_now=True + ) + harvester_type = models.CharField( + max_length=255, + help_text=_( + "Harvester class used to perform harvesting sessions. New harvester types " + "can be added by an admin by changing the main GeoNode `settings.py` file" + ), + choices=(((i, i) for i in get_setting("HARVESTER_CLASSES"))), + default=get_setting("HARVESTER_CLASSES")[0], + ) + harvester_type_specific_configuration = models.JSONField( + default=dict, + blank=True, + help_text=_( + "Configuration specific to each harvester type. Please consult GeoNode " + "documentation on harvesting for more info. This field is mandatory, so at " + "the very least an empty object (i.e. {}) must be supplied." + ), + ) + num_harvestable_resources = models.IntegerField(blank=True, default=0) + + class Meta: + # Note: name must be unique because we use it to create a periodic task for the harvester + constraints = [models.UniqueConstraint(fields=("name",), name="unique name")] + + def __str__(self): + return f"{self.name}({self.id})" + + @property + def latest_refresh_session(self): + try: + result = self.sessions.filter( + session_type=AsynchronousHarvestingSession.TYPE_DISCOVER_HARVESTABLE_RESOURCES + ).latest("started") + except AsynchronousHarvestingSession.DoesNotExist: + result = None + return result + + @property + def latest_harvesting_session(self): + try: + result = self.sessions.filter(session_type=AsynchronousHarvestingSession.TYPE_HARVESTING).latest("started") + except AsynchronousHarvestingSession.DoesNotExist: + result = None + return result + + def get_next_check_availability_dispatch_time(self) -> dt.datetime: + now = timezone.now() + latest_check = self.last_checked_availability + try: + next_schedule = latest_check + dt.timedelta(minutes=self.check_availability_frequency) + except TypeError: + result = now + else: + result = now if next_schedule < now else next_schedule + return result + + def get_next_refresh_session_dispatch_time(self) -> typing.Optional[dt.datetime]: + return self._get_next_dispatch_time(AsynchronousHarvestingSession.TYPE_DISCOVER_HARVESTABLE_RESOURCES) + + def get_next_harvesting_session_dispatch_time(self) -> typing.Optional[dt.datetime]: + return self._get_next_dispatch_time(AsynchronousHarvestingSession.TYPE_HARVESTING) + + def _get_next_dispatch_time(self, session_type: str) -> typing.Optional[dt.datetime]: + related_session_object_name, frequency_attribute = { + AsynchronousHarvestingSession.TYPE_DISCOVER_HARVESTABLE_RESOURCES: ( + "latest_refresh_session", + "refresh_harvestable_resources_update_frequency", + ), + AsynchronousHarvestingSession.TYPE_HARVESTING: ( + "latest_harvesting_session", + "harvesting_session_update_frequency", + ), + }[session_type] + latest_session: typing.Optional[AsynchronousHarvestingSession] = getattr(self, related_session_object_name) + frequency = getattr(self, frequency_attribute) + now = timezone.now() + if not self.scheduling_enabled: + result = None + elif latest_session is None: + result = now + else: + next_schedule = latest_session.started + dt.timedelta(minutes=frequency) + if next_schedule < now: + result = now + else: + result = next_schedule + return result + + def is_availability_check_due(self): + next_check_time = self.get_next_check_availability_dispatch_time() + now = timezone.now() + return next_check_time < now + + def is_harvestable_resources_refresh_due(self): + next_session_time = self.get_next_refresh_session_dispatch_time() + now = timezone.now() + return next_session_time < now + + def is_harvesting_due(self): + next_session_time = self.get_next_harvesting_session_dispatch_time() + now = timezone.now() + return next_session_time < now + + def clean(self): + """Perform model validation by inspecting fields that depend on each other. + + We validate the harvester type specific configuration by determining if it meets + the configured jsonschema (if any). + + """ + + try: + validate_worker_configuration(self.harvester_type, self.harvester_type_specific_configuration) + except jsonschema.exceptions.ValidationError as exc: + raise ValidationError(str(exc)) + + def update_availability(self, timeout_seconds: typing.Optional[int] = 5): + """Use the harvesting worker to check if the remote service is available""" + worker = self.get_harvester_worker() + self.last_checked_availability = timezone.now() + available = worker.check_availability(timeout_seconds=timeout_seconds) + self.remote_available = available + self.save() + return available + + def initiate_update_harvestable_resources(self): + should_continue, error_msg = self.worker_can_perform_action() + if should_continue: + self.status = self.STATUS_UPDATING_HARVESTABLE_RESOURCES + self.save() + refresh_session = AsynchronousHarvestingSession.objects.create( + harvester=self, session_type=AsynchronousHarvestingSession.TYPE_DISCOVER_HARVESTABLE_RESOURCES + ) + refresh_session.initiate() + else: + raise RuntimeError(error_msg) + + def initiate_perform_harvesting(self, harvestable_resource_ids: typing.Optional[typing.List[int]] = None): + should_continue, error_msg = self.worker_can_perform_action() + if should_continue: + self.status = self.STATUS_PERFORMING_HARVESTING + self.save() + harvesting_session = AsynchronousHarvestingSession.objects.create( + harvester=self, session_type=AsynchronousHarvestingSession.TYPE_HARVESTING + ) + harvesting_session.initiate(harvestable_resource_ids) + else: + raise RuntimeError(error_msg) + + def initiate_abort_update_harvestable_resources(self): + should_continue, error_msg = self.worker_can_perform_action(self.STATUS_UPDATING_HARVESTABLE_RESOURCES) + if should_continue: + self.status = self.STATUS_ABORTING_UPDATE_HARVESTABLE_RESOURCES + self.save() + self.latest_refresh_session.abort() + else: + raise RuntimeError(error_msg) + + def initiate_abort_perform_harvesting(self): + should_continue, error_msg = self.worker_can_perform_action(self.STATUS_PERFORMING_HARVESTING) + if should_continue: + self.status = self.STATUS_ABORTING_PERFORMING_HARVESTING + self.save() + self.latest_harvesting_session.abort() + else: + raise RuntimeError(error_msg) + + def get_harvester_worker(self) -> "BaseHarvesterWorker": # noqa + worker_class = import_string(self.harvester_type) + return worker_class.from_django_record(self) + + def worker_can_perform_action( + self, + target_status: typing.Optional[str] = STATUS_READY, + ) -> typing.Tuple[bool, str]: + if self.status != target_status: + error_message = ( + f"Harvester {self!r} cannot currently perform the desired action. Please wait until its status " + f"is reported as {target_status!r} before retrying." + ) + result = False + else: + result = True + error_message = "" + return result, error_message + + +class AsynchronousHarvestingSession(models.Model): + STATUS_PENDING = "pending" + STATUS_ON_GOING = "on-going" + STATUS_FINISHED_ALL_OK = "finished-all-ok" + STATUS_FINISHED_ALL_FAILED = "finished-all-failed" + STATUS_FINISHED_SOME_FAILED = "finished-some-failed" + STATUS_ABORTING = "aborting" + STATUS_ABORTED = "aborted" + STATUS_CHOICES = [ + (STATUS_PENDING, _("pending")), + (STATUS_ON_GOING, _("on-going")), + (STATUS_FINISHED_ALL_OK, _("finished-all-ok")), + (STATUS_FINISHED_ALL_FAILED, _("finished-all-failed")), + (STATUS_FINISHED_SOME_FAILED, _("finished-some-failed")), + (STATUS_ABORTING, _("aborting")), + (STATUS_ABORTED, _("aborted")), + ] + TYPE_HARVESTING = "harvesting" + TYPE_DISCOVER_HARVESTABLE_RESOURCES = "discover-harvestable-resources" + TYPE_CHOICES = [ + (TYPE_HARVESTING, _("harvesting")), + (TYPE_DISCOVER_HARVESTABLE_RESOURCES, _("discover-harvestable-resources")), + ] + session_type = models.CharField(max_length=50, choices=TYPE_CHOICES, editable=False) + started = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + ended = models.DateTimeField(null=True, blank=True) + harvester = models.ForeignKey(Harvester, on_delete=models.CASCADE, related_name="sessions") + status = models.CharField( + max_length=50, + choices=STATUS_CHOICES, + default=STATUS_PENDING, + editable=False, + ) + details = models.TextField( + blank=True, + ) + total_records_to_process = models.IntegerField( + default=0, editable=False, help_text=_("Number of records being processed in this session") + ) + records_done = models.IntegerField(default=0, help_text=_("Number of records that have already been processed")) + + @admin.display(description="Progress (%)") + def get_progress_percentage(self) -> int: + try: + result = (self.records_done / self.total_records_to_process) * 100 + except ZeroDivisionError: + result = 0 + return result + + def initiate(self, harvestable_resource_ids: typing.Optional[typing.List[int]] = None): + """Initiate the asynchronous process that performs the work related to this session.""" + # NOTE: below we are calling celery tasks using the method of creating a + # signature from the main celery app object in order to avoid having + # to import the `tasks` module, which would create circular + # dependency issues - this is a common celery pattern, described here: + # + # https://docs.celeryproject.org/en/stable/faq.html#can-i-call-a-task-by-name + # + # Also note that using `app.send_task()` does not seem to work in this case (which + # is mysterious, since the celery docs say it should work) + if self.session_type == self.TYPE_DISCOVER_HARVESTABLE_RESOURCES: + task_signature = celery_app.app.signature( + "geonode.harvesting.tasks.update_harvestable_resources", args=(self.pk,) + ) + elif self.session_type == self.TYPE_HARVESTING: + if harvestable_resource_ids is None: + task_signature = celery_app.app.signature( + "geonode.harvesting.tasks.harvesting_dispatcher", args=(self.pk,) + ) + else: + task_signature = celery_app.app.signature( + "geonode.harvesting.tasks.harvest_resources", args=(harvestable_resource_ids or [], self.pk) + ) + else: + raise RuntimeError("Invalid selection") + self.status = self.STATUS_PENDING + self.save() + task_signature.apply_async(args=(), expiration=30) + + def abort(self): + """Abort a pending or on-going session.""" + + # NOTE: We do not use celery's task revoke feature when aborting a session. This + # is an explicit design choice. The main reason being that we keep track of a session's + # state and want to know when it has finished. This is done by leveraging celery's `chord` + # feature, whereby the async tasks are executed in parallel and there is a final + # synchronization step when they are done that updates the session's state in the DB. + # Maintaining this synchronization step working OK together with revoking tasks would be + # harder to address. + if self.status == self.STATUS_PENDING: + self.status = self.STATUS_ABORTED + self.session_details = "Aborted" + elif self.status == self.STATUS_ON_GOING: + self.status = self.STATUS_ABORTING + else: + logger.debug("Session is not currently in an state that can be aborted, skipping...") + self.save() + + +class HarvestableResource(models.Model): + STATUS_READY = "ready" + STATUS_UPDATING_HARVESTABLE_RESOURCE = "updating-harvestable-resource" + STATUS_PERFORMING_HARVESTING = "harvesting-resource" + STATUS_CHOICES = [ + (STATUS_READY, _("ready")), + (STATUS_UPDATING_HARVESTABLE_RESOURCE, _("updating-harvestable-resource")), + (STATUS_PERFORMING_HARVESTING, _("harvesting-resource")), + ] + + unique_identifier = models.CharField( + max_length=255, + help_text=_( + "Identifier that allows referencing the resource on its remote service in " + "a unique fashion. This is usually automatically filled by the harvester " + "worker. The harvester worker needs to know how to either read " + "or generate this from each remote resource in order to be able to compare " + "the availability of resources between consecutive harvesting sessions." + ), + ) + status = models.CharField(max_length=50, choices=STATUS_CHOICES, default=STATUS_READY) + title = models.CharField(max_length=255) + abstract = models.TextField(max_length=2000, blank=True) + harvester = models.ForeignKey(Harvester, on_delete=models.CASCADE, related_name="harvestable_resources") + geonode_resource = models.ForeignKey("base.ResourceBase", null=True, on_delete=models.SET_NULL) + should_be_harvested = models.BooleanField(default=False) + last_updated = models.DateTimeField(auto_now=True) + last_refreshed = models.DateTimeField() + last_harvested = models.DateTimeField(null=True, blank=True) + last_harvesting_message = models.TextField(blank=True) + last_harvesting_succeeded = models.BooleanField(default=False) + remote_resource_type = models.CharField( + max_length=255, + help_text=_( + "Type of the resource in the remote service. Each harvester worker knows " + "how to fill this field, in accordance with the resources for which " + "harvesting is supported" + ), + blank=True, + ) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=("harvester", "unique_identifier"), name="unique_id_for_harvester"), + ] + + def delete(self, using=None, keep_parents=False): + delete_orphan_resource = self.harvester.delete_orphan_resources_automatically + worker = self.harvester.get_harvester_worker() + if self.geonode_resource is not None and delete_orphan_resource: + self.geonode_resource.delete() + if delete_orphan_resource: + worker.finalize_harvestable_resource_deletion(self) + return super().delete(using, keep_parents) + + +def validate_worker_configuration(worker_type: "BaseHarvesterWorker", worker_config: typing.Dict): # noqa + worker_class = import_string(worker_type) + schema = worker_class.get_extra_config_schema() + if schema is not None: + try: + jsonschema.validate(worker_config, schema) + except jsonschema.exceptions.SchemaError as exc: + raise RuntimeError(f"Invalid schema: {exc}") diff --git a/geonode/harvesting/resourcedescriptor.py b/geonode/harvesting/resourcedescriptor.py new file mode 100644 index 0000000..ca9cc82 --- /dev/null +++ b/geonode/harvesting/resourcedescriptor.py @@ -0,0 +1,94 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import dataclasses +import datetime as dt +import typing +import uuid + +from django.contrib.gis import geos + + +@dataclasses.dataclass() +class RecordDescriptionContact: + role: str + name: typing.Optional[str] = "" + organization: typing.Optional[str] = "" + position: typing.Optional[str] = "" + phone_voice: typing.Optional[str] = "" + phone_facsimile: typing.Optional[str] = "" + address_delivery_point: typing.Optional[str] = "" + address_city: typing.Optional[str] = "" + address_administrative_area: typing.Optional[str] = "" + address_postal_code: typing.Optional[str] = "" + address_country: typing.Optional[str] = "" + address_email: typing.Optional[str] = "" + + +@dataclasses.dataclass() +class RecordIdentification: + name: str + title: str + date: typing.Optional[dt.datetime] = None + date_type: typing.Optional[str] = None + originator: typing.Optional[RecordDescriptionContact] = None + place_keywords: typing.Optional[typing.List[str]] = None + other_keywords: typing.Optional[typing.Iterable] = None + license: typing.Optional[typing.List[str]] = None + abstract: typing.Optional[str] = "" + purpose: typing.Optional[str] = "" + status: typing.Optional[str] = "" + native_format: typing.Optional[str] = None + other_constraints: typing.Optional[str] = "" + topic_category: typing.Optional[str] = None + supplemental_information: typing.Optional[str] = "" + spatial_extent: typing.Optional[geos.Polygon] = None + lonlat_extent: typing.Optional[geos.Polygon] = None + temporal_extent: typing.Optional[typing.Tuple[dt.datetime, dt.datetime]] = None + + +@dataclasses.dataclass() +class RecordDistribution: + link_url: typing.Optional[str] = None + wms_url: typing.Optional[str] = None + wfs_url: typing.Optional[str] = None + wcs_url: typing.Optional[str] = None + thumbnail_url: typing.Optional[str] = None + download_url: typing.Optional[str] = None + embed_url: typing.Optional[str] = None + + +@dataclasses.dataclass() +class MapDescriptorParameters: + last_modified: dt.datetime + + +@dataclasses.dataclass() +class RecordDescription: + uuid: uuid.UUID + identification: RecordIdentification + distribution: RecordDistribution + point_of_contact: typing.Optional[RecordDescriptionContact] = None + author: typing.Optional[RecordDescriptionContact] = None + date_stamp: typing.Optional[dt.datetime] = None + reference_systems: typing.Optional[typing.List[str]] = None + data_quality: typing.Optional[str] = None + additional_parameters: typing.Optional[typing.Dict] = dataclasses.field(default_factory=dict) + language: typing.Optional[str] = None + character_set: typing.Optional[str] = None diff --git a/geonode/harvesting/signals.py b/geonode/harvesting/signals.py new file mode 100644 index 0000000..f125705 --- /dev/null +++ b/geonode/harvesting/signals.py @@ -0,0 +1,14 @@ +from django.db.models.signals import pre_delete +from django.dispatch import receiver +from geonode.harvesting.models import Harvester +from geonode.base.models import ResourceBase + + +@receiver(pre_delete, sender=Harvester) +def delete_orphan_resources_on_harvester_delete(sender, instance, **kwargs): + if instance.delete_orphan_resources_automatically: + resource_ids = instance.harvestable_resources.filter(geonode_resource__isnull=False).values_list( + "geonode_resource__id", flat=True + ) + + ResourceBase.objects.filter(id__in=resource_ids).delete() diff --git a/geonode/harvesting/tasks.py b/geonode/harvesting/tasks.py new file mode 100644 index 0000000..6384554 --- /dev/null +++ b/geonode/harvesting/tasks.py @@ -0,0 +1,717 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import math +import logging +import typing + +from celery import chord +from django.core.exceptions import ValidationError +from django.db.models import ( + F, + Value, +) +from django.db.models.functions import Concat +from django.utils import timezone +from django.conf import settings +from django.db import transaction + +from geonode.celery_app import app + +from . import models +from .harvesters import base + +logger = logging.getLogger(__name__) + + +@app.task(bind=True, queue="geonode", expires=30, time_limit=600, acks_late=False, ignore_result=False) +def harvesting_scheduler(self): + """Check whether any of the configured harvesters needs to be run or not. + + This function is called periodically by celery beat. It is configured to run every + `HARVESTER_SCHEDULER_FREQUENCY_MINUTES` minutes. This can be configured in the GeoNode + settings. The default value is 0.5, which means that this function is called every + thirty seconds. + + """ + + logger.debug("+++++ harvesting_dispatcher starting... +++++") + for harvester in models.Harvester.objects.all(): + if harvester.is_availability_check_due(): + logger.debug(f"{harvester.name} - Updating availability...") + available = harvester.update_availability() + logger.debug(f"{harvester.name} - is {'not ' if not available else ''}available") + else: + logger.debug(f"{harvester.name} - No need to update availability yet") + if harvester.scheduling_enabled: + logger.debug(f"{harvester.name} - scheduling_enabled: {harvester.scheduling_enabled!r}") + if harvester.is_harvestable_resources_refresh_due(): + logger.debug(f"{harvester.name} - Initiating update of harvestable resources...") + try: + harvester.initiate_update_harvestable_resources() + except RuntimeError: + logger.exception(msg=f"{harvester.name} - Could not initiate the update of harvestable resources") + else: + logger.debug(f"{harvester.name} - No need to update harvestable resources yet") + if harvester.is_harvesting_due(): + logger.debug(f"{harvester.name} - initiating harvesting...") + try: + harvester.initiate_perform_harvesting() + except RuntimeError: + logger.exception(msg=f"{harvester.name} - Could not initiate harvesting") + else: + logger.debug(f"{harvester.name} - No need to harvest yet") + else: + logger.debug(f"{harvester.name} - Scheduling is disabled for this harvester, skipping...") + logger.debug("+++++ harvesting_dispatcher ending... +++++") + + +@app.task( + bind=True, + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, + ignore_result=False, +) +def harvesting_dispatcher(self, harvesting_session_id: int): + """Perform harvesting asynchronously. + + This function kicks-off a harvesting session for the input harvester id. + It defines an async workflow that results in the harvestable resources being + checked and harvested onto the local GeoNode. + + The implementation briefly consists of: + + - start a harvesting session + - determine which of the known harvestable resources have to be harvested + - schedule each harvestable resource to be harvested asynchronously + - when all resources have been harvested, finish the harvesting session + + Internally, the code uses celery's `chord` workflow primitive. This is used to + allow the harvesting of individual resources to be done in parallel (as much + as the celery worker config allows for) and still have a final synchronization + step, once all resources are harvested, that finalizes the harvesting session. + + """ + + session = models.AsynchronousHarvestingSession.objects.get(pk=harvesting_session_id) + harvester = session.harvester + harvestable_resources = list( + harvester.harvestable_resources.filter(should_be_harvested=True).values_list("id", flat=True) + ) + if len(harvestable_resources) > 0: + harvest_resources.apply_async(args=(harvestable_resources, harvesting_session_id)) + else: + message = "harvesting_dispatcher - Nothing to do" + logger.debug(message) + finish_asynchronous_session( + harvesting_session_id, models.AsynchronousHarvestingSession.STATUS_FINISHED_ALL_OK, final_details=message + ) + + +@app.task( + bind=True, + queue="geonode", + expires=120, + time_limit=600, + acks_late=False, + ignore_result=False, +) +def harvest_resources( + self, + harvestable_resource_ids: typing.List[int], + harvesting_session_id: int, +): + """Harvest a list of remote resources that all belong to the same harvester.""" + session = models.AsynchronousHarvestingSession.objects.get(pk=harvesting_session_id) + if session.status == session.STATUS_ABORTED: + logger.debug("Session has been aborted, skipping...") + return + + if not harvestable_resource_ids: + logger.debug("harvest_resources - Nothing to do...") + finish_asynchronous_session( + harvesting_session_id, + models.AsynchronousHarvestingSession.STATUS_FINISHED_ALL_OK, + final_details="No resources to harvest", + ) + return + + harvester = session.harvester + + if not harvester.update_availability(): + message = f"Skipping harvesting for harvester {harvester.name!r} because remote {harvester.remote_url!r} is unavailable" + logger.warning(message) + finish_asynchronous_session(harvesting_session_id, session.STATUS_FINISHED_ALL_FAILED, final_details=message) + return + + harvester.status = harvester.STATUS_PERFORMING_HARVESTING + harvester.save() + + session.status = session.STATUS_ON_GOING + session.total_records_to_process = len(harvestable_resource_ids) + session.save() + + # Definition of the expiration time + task_dynamic_expiration = calculate_dynamic_expiration(len(harvestable_resource_ids)) + + harvestable_resources_limit = settings.CHUNK_SIZE + + if len(harvestable_resource_ids) <= harvestable_resources_limit: + # No chunking, just one chord for all resources + resource_tasks = [ + _harvest_resource.signature((rid, harvesting_session_id)).set(expires=task_dynamic_expiration) + for rid in harvestable_resource_ids + ] + finalizer = ( + _finish_harvesting.signature((harvesting_session_id,), immutable=True) + .on_error(_handle_harvesting_error.signature(kwargs={"harvesting_session_id": harvesting_session_id})) + .set(expires=task_dynamic_expiration) + ) + harvesting_workflow = chord(resource_tasks, body=finalizer) + transaction.on_commit(lambda: harvesting_workflow.apply_async()) + + else: + # Chunk the resource IDs only, NOT the Celery tasks + chunks = list(chunked(harvestable_resource_ids)) + max_parallel_chunks = settings.MAX_PARALLEL_QUEUE_CHUNKS + chunk_groups = list(chunked(chunks, max_parallel_chunks)) + + # Estimate dynamic time limit + dynamic_time_limit = calculate_dynamic_time_limit() + + logger.debug(f"Chunk groups: {chunk_groups}") + + transaction.on_commit( + lambda: queue_next_chunk_batch.apply_async( + args=(chunk_groups, harvesting_session_id), + kwargs={ + "batch_index": 0, + "dynamic_expiration": task_dynamic_expiration, + "dynamic_time_limit": dynamic_time_limit, + }, + expires=task_dynamic_expiration, + time_limit=dynamic_time_limit, + immutable=True, + ) + ) + + +@app.task( + bind=True, + queue="geonode", + time_limit=600, + acks_late=False, + ignore_result=False, +) +def _harvest_resource(self, harvestable_resource_id: int, harvesting_session_id: int): + """ + Harvest a single resource from the input harvestable resource id + + NOTE: + The expiration time (`expires`) of this task is set dynamically when the task + is created or chained, not via the decorator. This allows the expiration to be + calculated based on the current workload or batch size. + """ + session = models.AsynchronousHarvestingSession.objects.get(pk=harvesting_session_id) + if session.status != session.STATUS_ABORTING: + harvestable_resource = models.HarvestableResource.objects.get(pk=harvestable_resource_id) + worker: base.BaseHarvesterWorker = harvestable_resource.harvester.get_harvester_worker() + harvested_resource_info = worker.get_resource(harvestable_resource) + now_ = timezone.now() + if harvested_resource_info is not None: + if worker.should_copy_resource(harvestable_resource): + copied_path = worker.copy_resource(harvestable_resource, harvested_resource_info) + if copied_path is not None: + harvested_resource_info.copied_resources.append(copied_path) + try: + worker.update_geonode_resource( + harvested_resource_info, + harvestable_resource, + ) + result = True + details = "" + except (RuntimeError, ValidationError) as exc: + logger.error(msg="Unable to update geonode resource") + result = False + details = str(exc) + harvesting_message = ( + f"{harvestable_resource.title}({harvestable_resource_id}) - {'Success' if result else details}" + ) + update_asynchronous_session( + harvesting_session_id, + additional_processed_records=1 if result else 0, + additional_details=harvesting_message, + ) + harvestable_resource.last_harvesting_message = f"{now_} - {harvesting_message}" + harvestable_resource.last_harvesting_succeeded = result + else: + harvestable_resource.last_harvesting_message = f"{now_}Harvesting failed" + harvestable_resource.last_harvesting_succeeded = False + harvestable_resource.last_harvested = now_ + harvestable_resource.save() + else: + message = f"Skipping harvesting of resource {harvestable_resource_id} since the " f"session has been aborted" + update_asynchronous_session(harvesting_session_id, additional_details=message) + logger.debug(message) + + +@app.task( + bind=True, + queue="geonode", + time_limit=600, + acks_late=False, + ignore_result=False, +) +def _finish_harvesting(self, harvesting_session_id: int): + """ + Finalize the harvesting session by setting its final status and logging the result. + + NOTE: + The expiration time is set dynamically when this task is scheduled, + so the decorator does NOT specify an expires parameter. + """ + session = models.AsynchronousHarvestingSession.objects.get(pk=harvesting_session_id) + harvester = session.harvester + if session.status == session.STATUS_ABORTING: + message = "Harvesting session aborted by user" + final_status = session.STATUS_ABORTED + else: + message = "Harvesting completed successfully!" + final_status = session.STATUS_FINISHED_ALL_OK + finish_asynchronous_session(harvesting_session_id, final_status=final_status, final_details=message) + logger.debug(f"(harvester: {harvester.pk!r} - session: {harvesting_session_id!r}) " f"{message}") + + +@app.task(bind=True, queue="geonode", time_limit=600, acks_late=False, ignore_result=False) +def _finish_harvesting_chunk(self, _results, harvesting_session_id: int): + """ + Optionally log chunk completion, but do NOT change final status here + + NOTE: + The expiration time is set dynamically when this task is scheduled, + so the decorator does NOT specify an expires parameter. + """ + logger.debug(f"Chunk finished for session {harvesting_session_id} with {len(_results)} resources.") + + +@app.task( + bind=True, + queue="geonode", + time_limit=600, + acks_late=False, + ignore_result=False, +) +def queue_next_chunk_batch( + self, + chunk_groups: list, + harvesting_session_id: int, + batch_index: int = 0, + dynamic_expiration: int | None = None, + dynamic_time_limit: int | None = None, +): + """ + Queue the next batch of chunk (group of harvestable resources) + chords (default is 2 at a time). + + NOTE: + The dynamic_expiration parameter controls the dynamic expiration time for all subtasks and chords. + """ + + if dynamic_expiration is None: + dynamic_expiration = 600 # fallback expiration (10 minutes) + + if batch_index >= len(chunk_groups): + logger.debug(f"All batches completed for session {harvesting_session_id}") + return + + logger.debug( + f"Queueing batch {batch_index + 1}/{len(chunk_groups)} for session {harvesting_session_id} with expires={dynamic_expiration}" + ) + + current_chunk_group = chunk_groups[batch_index] + chords_in_batch = [] + + for chunk in current_chunk_group: + resource_tasks = [ + _harvest_resource.s(rid, harvesting_session_id).set(expires=dynamic_expiration) for rid in chunk + ] + + chunk_finalizer = _finish_harvesting_chunk.s(harvesting_session_id).set(expires=dynamic_expiration) + chords_in_batch.append(chord(resource_tasks, body=chunk_finalizer)) + + # Add global finalizer only to the last batch + is_last_batch = batch_index == len(chunk_groups) - 1 + if is_last_batch: + global_finalizer = ( + _finish_harvesting.s(harvesting_session_id) + .on_error(_handle_harvesting_error.s(harvesting_session_id)) + .set(expires=dynamic_expiration) + ) + chords_in_batch.append(global_finalizer) + + next_batch = queue_next_chunk_batch.s( + chunk_groups, + harvesting_session_id, + batch_index + 1, + dynamic_expiration=dynamic_expiration, + dynamic_time_limit=dynamic_time_limit, + ).set(expires=dynamic_expiration, time_limit=dynamic_time_limit, immutable=True) + + chord(chords_in_batch, body=next_batch).apply_async() + + +@app.task( + bind=True, + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, + ignore_result=False, +) +def _handle_harvesting_error(self, task_id, *args, **kwargs): + """Ensure harvester status is set back to `ready` in case of an error.""" + # FIXME NOTE: Unfortunately, we don't seem to have a nice way of knowing what + # caused the error of a harvesting session. Maybe I just don't understand how + # this is supposed to work (quite possible) but apparently this is a bug with + # celery (or maybe just a case of unclear documentation): + # + # https://github.com/celery/celery/issues/3709#issuecomment-695809092 + # + # The celery docs do not match the code: + # + # https://docs.celeryproject.org/en/stable/userguide/canvas.html#error-handling + # + # Celery docs claim that the error handler receives `request, exc, traceback` as its + # parameters. However, what we are indeed receiving is `self, task_id, args, kwargs`. + # + # Moreover, there does not seem to be a way to get the actual traceback of the failed task. + # Below we are instantiating the result object, which is gotten from the input `task_id`, + # However, when checking the result's `traceback` attribute, it shows up empty. + # + logger.debug( + "Inside handle_harvesting_error task ---------------------------------------------------------------------------------------------" + ) + logger.debug(f"locals before getting the result: {locals()}") + result = self.app.AsyncResult(str(task_id)) + logger.debug(f"locals after getting the result: {locals()}") + logger.debug(f"state: {result.state}") + logger.debug(f"result: {result.result}") + logger.debug(f"traceback: {result.traceback}") + logger.debug(f"harvesting task with kwargs: {kwargs} has failed") + session = models.AsynchronousHarvestingSession.objects.get(pk=kwargs["harvesting_session_id"]) + details = f"state: {result.state}\nresult: {result.result}\ntraceback: {result.traceback}" + finish_asynchronous_session( + kwargs["harvesting_session_id"], session.STATUS_FINISHED_SOME_FAILED, final_details=details + ) + + +@app.task( + bind=True, + # name='geonode.harvesting.tasks.check_harvester_available', + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, +) +def check_harvester_available(self, harvester_id: int): + harvester = models.Harvester.objects.get(pk=harvester_id) + available = harvester.update_availability() + logger.info(f"Harvester {harvester!r}: remote server is " f"{'' if available else 'not '}available") + + +@app.task( + bind=True, + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, + ignore_result=False, +) +def update_harvestable_resources(self, refresh_session_id: int): + # NOTE: we are able to implement batch discovery of existing harvestable resources + # because we want to know about all of them. We are not able to batch harvesting + # of resources because these have potentially been individually selected by the + # user, which means we are not interested in all of them + session = models.AsynchronousHarvestingSession.objects.get(pk=refresh_session_id) + if session.status != session.STATUS_ABORTED: + session.status = session.STATUS_ON_GOING + session.save() + harvester = session.harvester + if harvester.update_availability(): + harvester.status = harvester.STATUS_UPDATING_HARVESTABLE_RESOURCES + harvester.save() + worker = harvester.get_harvester_worker() + try: + num_resources = worker.get_num_available_resources() + except (NotImplementedError, base.HarvestingException) as exc: + _handle_harvestable_resources_update_error( + self.request.id, refresh_session_id=refresh_session_id, raised_exception=exc + ) + else: + harvester.num_harvestable_resources = num_resources + harvester.save() + session.total_records_to_process = num_resources + session.save() + page_size = 10 + total_pages = math.ceil(num_resources / page_size) + batches = [] + for page in range(total_pages): + batches.append( + _update_harvestable_resources_batch.signature( + args=(refresh_session_id, page, page_size), + ) + ) + update_finalizer = _finish_harvestable_resources_update.signature( + args=(refresh_session_id,), immutable=True + ).on_error( + _handle_harvestable_resources_update_error.signature( + kwargs={"refresh_session_id": refresh_session_id} + ) + ) + update_workflow = chord(batches, body=update_finalizer) + update_workflow.apply_async(args=(), expiration=30) + else: + finish_asynchronous_session( + refresh_session_id, session.STATUS_FINISHED_ALL_FAILED, final_details="Harvester is not available" + ) + else: + logger.debug("Session has been aborted, skipping...") + + +@app.task( + bind=True, + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, + ignore_result=False, +) +def _update_harvestable_resources_batch(self, refresh_session_id: int, page: int, page_size: int): + session = models.AsynchronousHarvestingSession.objects.get(pk=refresh_session_id) + if session.status == session.STATUS_ON_GOING: + harvester = session.harvester + worker = harvester.get_harvester_worker() + offset = page * page_size + try: + found_resources = worker.list_resources(offset) + except base.HarvestingException: + logger.exception("Could not retrieve list of remote resources.") + else: + processed = 0 + for remote_resource in found_resources: + resource, created = models.HarvestableResource.objects.get_or_create( + harvester=harvester, + unique_identifier=remote_resource.unique_identifier, + title=remote_resource.title, + defaults={ + "should_be_harvested": harvester.harvest_new_resources_by_default, + "remote_resource_type": remote_resource.resource_type, + "last_refreshed": timezone.now(), + }, + ) + processed += 1 + # NOTE: make sure to save the resource because we need to have its + # `last_updated` property be refreshed - this is done in order to be able + # to compare when a resource has been found + resource.last_refreshed = timezone.now() + resource.save() + update_asynchronous_session(refresh_session_id, additional_processed_records=processed) + else: + logger.info("The refresh session has been asked to abort, so skipping...") + + +@app.task( + bind=True, + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, + ignore_result=False, +) +def _finish_harvestable_resources_update(self, refresh_session_id: int): + session = models.AsynchronousHarvestingSession.objects.get(pk=refresh_session_id) + harvester = session.harvester + if session.status == session.STATUS_ABORTING: + message = "Refresh session aborted by user" + finish_asynchronous_session(refresh_session_id, session.STATUS_ABORTED, final_details=message) + else: + message = "Harvestable resources successfully refreshed" + finish_asynchronous_session(refresh_session_id, session.STATUS_FINISHED_ALL_OK, final_details=message) + if harvester.last_checked_harvestable_resources is not None: + _delete_stale_harvestable_resources(harvester) + now_ = timezone.now() + harvester.refresh_from_db() + harvester.last_checked_harvestable_resources = now_ + harvester.last_check_harvestable_resources_message = f"{now_} - {message}" + harvester.save() + + +@app.task( + bind=True, + queue="geonode", + expires=30, + time_limit=600, + acks_late=False, + ignore_result=False, +) +def _handle_harvestable_resources_update_error(self, task_id, *args, **kwargs): + logger.debug("Inside _handle_harvestable_resources_update_error -----------------------------------------") + result = self.app.AsyncResult(str(task_id)) + logger.debug(f"locals: {locals()}") + logger.debug(f"state: {result.state}") + logger.debug(f"result: {result.result}") + logger.debug(f"traceback: {result.traceback}") + result = self.app.AsyncResult(str(task_id)) + session = models.AsynchronousHarvestingSession.objects.get(pk=kwargs["refresh_session_id"]) + harvester = session.harvester + now_ = timezone.now() + harvester.last_checked_harvestable_resources = now_ + harvester.last_check_harvestable_resources_message = ( + f"{now_} - There was an error retrieving information on available " + f"resources: {result.traceback} - {args} {kwargs}" + f"Please check the logs" + ) + harvester.save() + details = f"state: {result.state}\nresult: {result.result}\ntraceback: {result.traceback}" + finish_asynchronous_session( + kwargs["refresh_session_id"], + final_status=models.AsynchronousHarvestingSession.STATUS_FINISHED_SOME_FAILED, + final_details=details, + ) + + +def _delete_stale_harvestable_resources(harvester: models.Harvester): + """Delete harvestable resources that haven't been found on the current refresh. + + If a harvestable resource was not refreshed since the last check, it this means + it has not been found this time around - maybe it is not available anymore, or + maybe the user just changed the harvester parameters. Either way, the harvestable + resource is no longer relevant, and therefore can be deleted. + + When a harvestable resource is deleted, it may leave a corresponding GeoNode + resource orphan. The corresponding GeoNode resource is the one that had been + created locally as a result of harvesting information of the HarvestableResource + on the remote server by the harvester worker. Depending on the value of the + corresponding `models.Harvester.delete_orphan_resources_automatically`, the GeoNode + resource may also be deleted as a result of running this method. + + """ + + previously_checked_at = harvester.last_checked_harvestable_resources + logger.debug(f"last checked at: {previously_checked_at}") + logger.debug(f"now: {timezone.now()}") + to_remove = models.HarvestableResource.objects.filter( + harvester=harvester, last_refreshed__lte=previously_checked_at + ) + for harvestable_resource in to_remove: + # NOTE: Iterating on each resource and calling its `delete()` method instead of + # just calling `delete()` on the queryset because this way we can be sure that + # the instance's `delete()` method is called (django may not call + # `instance.delete()` when using the queryset directly, for performance + # reasons). This is because `HarvestableResource.delete()` has the custom logic + # to check whether the related GeoNode resource should also be deleted or not + harvestable_resource.delete() + + +def finish_asynchronous_session( + session_id: int, + final_status: str, + final_details: typing.Optional[str] = None, + additional_processed_records: typing.Optional[int] = None, +) -> None: + """Finish the asynchronous session and also reset the harvester status.""" + update_kwargs = { + "ended": timezone.now(), + "status": final_status, + } + if additional_processed_records is not None: + update_kwargs["records_done"] = F("records_done") + additional_processed_records + if final_details is not None: + update_kwargs["details"] = Concat("details", Value(f"\n{final_details}")) + models.AsynchronousHarvestingSession.objects.filter(id=session_id).update(**update_kwargs) + models.Harvester.objects.filter(sessions__pk=session_id).update(status=models.Harvester.STATUS_READY) + + +def update_asynchronous_session( + session_id: int, + total_records_to_process: typing.Optional[int] = None, + additional_processed_records: typing.Optional[int] = None, + additional_details: typing.Optional[str] = None, +) -> None: + update_kwargs = {} + if total_records_to_process is not None: + update_kwargs["total_records_to_process"] = total_records_to_process + if additional_processed_records is not None: + update_kwargs["records_done"] = F("records_done") + additional_processed_records + if additional_details is not None: + update_kwargs["details"] = Concat("details", Value(f"\n{additional_details}")) + models.AsynchronousHarvestingSession.objects.filter(id=session_id).update(**update_kwargs) + + +def calculate_dynamic_expiration( + num_resources: int, + estimated_duration_per_resource: int = 20, + buffer_time: int = 300, +) -> int: + """ + Calculate a dynamic expiration time (in seconds) + depending on the harvestable resources + + Args: + num_resources (int): Number of resources/tasks. + estimated_duration_per_resource (int): Estimated time per resource in seconds. + buffer_time (int): Additional buffer time in seconds. + + Returns: + estimation time in seconds. + """ + return num_resources * estimated_duration_per_resource + buffer_time + + +def calculate_dynamic_time_limit( + estimated_duration_per_resource: int = 20, + buffer_time: int = 300, +) -> int: + """ + Calculate a dynamic time_limit (in seconds) + depending on the chunk and batch size + + Args: + batch_size (int): Number of resources that are inserted in the queue at once. + estimated_duration_per_resource (int): Estimated time per resource in seconds. + buffer_time (int): Additional buffer time in seconds. + + Returns: + time_limit in seconds. + """ + + chunk_size = settings.CHUNK_SIZE + max_parallel_queue_chunks = settings.MAX_PARALLEL_QUEUE_CHUNKS + + batch_size = chunk_size * max_parallel_queue_chunks + estimated_time_limit = batch_size * estimated_duration_per_resource + buffer_time + + return estimated_time_limit + + +def chunked(iterable, chunk_size=100): + for i in range(0, len(iterable), chunk_size): + yield iterable[i : i + chunk_size] diff --git a/geonode/harvesting/templates/harvesting/harvesters_geonode_get_records.xml b/geonode/harvesting/templates/harvesting/harvesters_geonode_get_records.xml new file mode 100644 index 0000000..f5bc2f7 --- /dev/null +++ b/geonode/harvesting/templates/harvesting/harvesters_geonode_get_records.xml @@ -0,0 +1,17 @@ + + + full + + \ No newline at end of file diff --git a/geonode/harvesting/tests/__init__.py b/geonode/harvesting/tests/__init__.py new file mode 100644 index 0000000..6b4db60 --- /dev/null +++ b/geonode/harvesting/tests/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/harvesting/tests/factories.py b/geonode/harvesting/tests/factories.py new file mode 100644 index 0000000..1827c95 --- /dev/null +++ b/geonode/harvesting/tests/factories.py @@ -0,0 +1,53 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + + +import uuid +import datetime +from geonode.harvesting import resourcedescriptor +from geonode.harvesting.harvesters.base import HarvestedResourceInfo, BriefRemoteResource + +contact_example = resourcedescriptor.RecordDescriptionContact(role="role", name="Test") +identification_example = resourcedescriptor.RecordIdentification( + name="Test", + title="Test", + date=datetime.datetime.now(), + date_type="type", + originator=contact_example, + place_keywords=["keyword"], + other_keywords=("test",), + license=["test"], +) +distribution_example = resourcedescriptor.RecordDistribution() +resource_description_example = resourcedescriptor.RecordDescription( + uuid=uuid.uuid4(), + point_of_contact=contact_example, + author=contact_example, + date_stamp=datetime.datetime.now(), + identification=identification_example, + distribution=distribution_example, +) + +resource_info_example = HarvestedResourceInfo( + resource_descriptor=resource_description_example, additional_information={} +) + +brief_remote_resource_example = BriefRemoteResource( + unique_identifier="id", title="Test", abstract="", resource_type="Layer" +) diff --git a/geonode/harvesting/tests/harvesters/__init__.py b/geonode/harvesting/tests/harvesters/__init__.py new file mode 100644 index 0000000..5d1d824 --- /dev/null +++ b/geonode/harvesting/tests/harvesters/__init__.py @@ -0,0 +1,18 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/harvesting/tests/harvesters/base.py b/geonode/harvesting/tests/harvesters/base.py new file mode 100644 index 0000000..4b6dd56 --- /dev/null +++ b/geonode/harvesting/tests/harvesters/base.py @@ -0,0 +1,86 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import datetime +from django.contrib.auth import get_user_model +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.harvesting.models import Harvester, HarvestableResource +from geonode.harvesting.tests.harvesters.test_harvester import TestHarvester +from geonode.layers.models import Dataset + + +class TestBaseHarvester(GeoNodeBaseTestSupport): + """ + Test Base harvester + """ + + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.tests.harvesters.test_harvester.TestHarvester" + + def setUp(self): + super().setUp() + self.worker = TestHarvester(remote_url=self.remote_url, harvester_id=1) + + def test_worker_from_harvester(self): + """ + Test worker that generated from harvester + """ + harvester = Harvester.objects.create( + remote_url=self.remote_url, name=self.name, default_owner=self.user, harvester_type=self.harvester_type + ) + worker = harvester.get_harvester_worker() + self.assertEqual(worker.__class__, TestHarvester) + self.assertEqual(worker.remote_url, self.remote_url) + self.assertEqual(harvester.default_owner, self.user) + + def test_worker_from_django_record(self): + """ + Test worker that generated from worker using harvester record + """ + harvester = Harvester.objects.create( + remote_url=self.remote_url, name=self.name, default_owner=self.user, harvester_type=self.harvester_type + ) + worker = TestHarvester.from_django_record(harvester) + self.assertEqual(worker.__class__, TestHarvester) + self.assertEqual(worker.remote_url, self.remote_url) + self.assertEqual(harvester.default_owner, self.user) + + def test_worker_methods(self): + """ + Test functions in worker + """ + self.assertEqual(self.worker.remote_url, self.remote_url) + self.assertEqual(self.worker.harvester_id, 1) + self.assertTrue(self.worker.allows_copying_resources) + self.assertTrue(self.worker.check_availability()) + self.assertEqual(self.worker.get_num_available_resources(), 1) + self.assertEqual(len(self.worker.list_resources()), 1) + self.assertEqual(self.worker.get_geonode_resource_type("type"), Dataset) + + harvestable_resource = HarvestableResource( + unique_identifier="1", + title="Test Resource", + harvester=Harvester( + remote_url=self.remote_url, name=self.name, default_owner=self.user, harvester_type=self.harvester_type + ), + last_refreshed=datetime.datetime.now(), + ) + self.assertIsNone(self.worker.get_resource(harvestable_resource, 1)) diff --git a/geonode/harvesting/tests/harvesters/geonode.py b/geonode/harvesting/tests/harvesters/geonode.py new file mode 100644 index 0000000..5ffec7b --- /dev/null +++ b/geonode/harvesting/tests/harvesters/geonode.py @@ -0,0 +1,172 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from mock import patch +from django.contrib.auth import get_user_model +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.harvesting.models import Harvester +from geonode.harvesting.harvesters.geonodeharvester import GeonodeLegacyHarvester, GeoNodeResourceType +from geonode.harvesting.harvesters.base import BriefRemoteResource + +test_resources = { + GeoNodeResourceType.DATASET: 1, + GeoNodeResourceType.DOCUMENT: 2, + GeoNodeResourceType.MAP: 3, +} + + +def geonode_get_total_records(cls, resource_type: GeoNodeResourceType): + """ + Fake _get_total_records function on GeonodeLegacyHarvester + """ + return test_resources[resource_type] + + +def geonode_list_resources_by_type(cls, resource_type: GeoNodeResourceType, offset: int): + """ + Fake _list_resources_by_type function on GeonodeLegacyHarvester + """ + return [ + BriefRemoteResource( + unique_identifier="ID", + title="Title", + resource_type=resource_type.value, + ) + ] + + +class TestGeonodeHarvester(GeoNodeBaseTestSupport): + """ + Test GeonodeLegacyHarvester + """ + + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester" + + def setUp(self): + super().setUp() + self.worker = GeonodeLegacyHarvester(remote_url=self.remote_url, harvester_id=1) + + def test_base_api_url(self): + """ + Test the return of base_api_url + """ + self.assertEqual(self.worker.base_api_url, f"{self.remote_url}/api") + + def test_allows_copying_resources(self): + """ + Test the return of allows_copying_resources + """ + self.assertTrue(self.worker.allows_copying_resources) + + @patch("geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester._get_total_records", geonode_get_total_records) + def test_get_num_available_resources_by_type(self): + """ + Test function of _get_num_available_resources_by_type + """ + worker = GeonodeLegacyHarvester(remote_url=self.remote_url, harvester_id=1) + self.assertEqual(worker._get_num_available_resources_by_type(), test_resources) + self.assertEqual( + worker._get_total_records(GeoNodeResourceType.DATASET), test_resources[GeoNodeResourceType.DATASET] + ) + self.assertEqual( + worker._get_total_records(GeoNodeResourceType.DOCUMENT), test_resources[GeoNodeResourceType.DOCUMENT] + ) + self.assertEqual(worker._get_total_records(GeoNodeResourceType.MAP), test_resources[GeoNodeResourceType.MAP]) + + @patch("geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester._get_total_records", geonode_get_total_records) + def test_get_num_available_resources(self): + """ + Test function of get_num_available_resources for each of type in GeonodeLegacyHarvester + """ + params = {"remote_url": self.remote_url, "harvester_id": 1} + # test worker that harvest all type + worker = GeonodeLegacyHarvester(**params) + self.assertEqual(worker.get_num_available_resources(), 6) + + # test worker with skip document + worker = GeonodeLegacyHarvester(**params, harvest_documents=False) + self.assertEqual(worker.get_num_available_resources(), 4) + + # test worker with skip layer + worker = GeonodeLegacyHarvester(**params, harvest_datasets=False) + self.assertEqual(worker.get_num_available_resources(), 5) + + # test worker with skip maps + worker = GeonodeLegacyHarvester(**params, harvest_maps=False) + self.assertEqual(worker.get_num_available_resources(), 3) + + @patch( + "geonode.harvesting.harvesters.geonode.GeonodeLegacyHarvester._list_resources_by_type", + geonode_list_resources_by_type, + ) + def test_list_resources_by_type(self): + """ + Test _list_resources_by_type function for every type in GeonodeLegacyHarvester + """ + self.assertEqual( + self.worker._list_resources_by_type(GeoNodeResourceType.DATASET, 0)[0].resource_type, + GeoNodeResourceType.DATASET.value, + ) + self.assertEqual(self.worker._list_dataset_resources(1)[0].resource_type, GeoNodeResourceType.DATASET.value) + + self.assertEqual( + self.worker._list_resources_by_type(GeoNodeResourceType.DOCUMENT, 0)[0].resource_type, + GeoNodeResourceType.DOCUMENT.value, + ) + self.assertEqual(self.worker._list_document_resources(1)[0].resource_type, GeoNodeResourceType.DOCUMENT.value) + + self.assertEqual( + self.worker._list_resources_by_type(GeoNodeResourceType.MAP, 0)[0].resource_type, + GeoNodeResourceType.MAP.value, + ) + self.assertEqual(self.worker._list_map_resources(1)[0].resource_type, GeoNodeResourceType.MAP.value) + + def test_extract_unique_identifier(self): + """ + Test _extract_unique_identifier function + """ + self.assertEqual(self.worker._extract_unique_identifier({"id": 1}), 1) + + def test_worker_from_django_record(self): + """ + Test worker that genearted by harvester + """ + harvester = Harvester.objects.create( + remote_url=self.remote_url, + name=self.name, + default_owner=self.user, + harvester_type=self.harvester_type, + harvester_type_specific_configuration={ + "harvest_documents": False, + "harvest_datasets": True, + "resource_title_filter": "", + }, + ) + worker = GeonodeLegacyHarvester.from_django_record(harvester) + self.assertEqual(worker.__class__, GeonodeLegacyHarvester) + self.assertEqual(worker.remote_url, self.remote_url) + self.assertEqual(harvester.default_owner, self.user) + + self.assertFalse(worker.harvest_documents) + self.assertTrue(worker.harvest_datasets) + self.assertTrue(worker.harvest_maps) + self.assertEqual(worker.resource_name_filter, "") diff --git a/geonode/harvesting/tests/harvesters/test_harvester.py b/geonode/harvesting/tests/harvesters/test_harvester.py new file mode 100644 index 0000000..19d0fa3 --- /dev/null +++ b/geonode/harvesting/tests/harvesters/test_harvester.py @@ -0,0 +1,56 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import typing +from geonode.harvesting.harvesters.base import BaseHarvesterWorker, BriefRemoteResource, HarvestedResourceInfo +from geonode.harvesting.models import Harvester, HarvestableResource +from geonode.layers.models import Dataset +from geonode.harvesting.tests.factories import brief_remote_resource_example + + +class TestHarvester(BaseHarvesterWorker): + """ + Override base harvester as test harvester + """ + + @property + def allows_copying_resources(self) -> bool: + return True + + @classmethod + def from_django_record(cls, record: Harvester): + return cls(record.remote_url, record.id) + + def get_num_available_resources(self) -> int: + return 1 + + def list_resources(self, offset: typing.Optional[int] = 0) -> typing.List[BriefRemoteResource]: + return [brief_remote_resource_example] + + def check_availability(self, timeout_seconds: typing.Optional[int] = 5) -> bool: + return True + + def get_resource( + self, harvestable_resource: "HarvestableResource", harvesting_session_id: int # noqa + ) -> typing.Optional[HarvestedResourceInfo]: + return None + + def get_geonode_resource_type(self, remote_resource_type: str): + """Return resource type class from resource type string.""" + return Dataset diff --git a/geonode/harvesting/tests/test_admin.py b/geonode/harvesting/tests/test_admin.py new file mode 100644 index 0000000..610a8b7 --- /dev/null +++ b/geonode/harvesting/tests/test_admin.py @@ -0,0 +1,147 @@ +from unittest import mock + +from django.contrib.admin.sites import AdminSite +from django.urls import reverse +from django.contrib.auth import get_user_model +from django.test import RequestFactory +from rest_framework import status + +from geonode.tests.base import GeoNodeBaseTestSupport + +from .. import admin, models + + +class HarvesterAdminTestCase(GeoNodeBaseTestSupport): + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" + + def setUp(self): + self.factory = RequestFactory() + self.user = get_user_model().objects.get(username="admin") + self.client.login(username="admin", password="admin") + + self.harvester = models.Harvester.objects.create( + remote_url="http://fake1.com", + name="harvester1", + default_owner=self.user, + harvester_type=self.harvester_type, + ) + + def test_get_form_returns_current_user(self): + request = self.factory.post(reverse("admin:harvesting_harvester_add")) + request.user = self.user + model_admin = admin.HarvesterAdmin(model=models.Harvester, admin_site=AdminSite()) + form = model_admin.get_form(request) + self.assertEqual(form.base_fields["default_owner"].initial.username, self.user.username) + + def test_add_harvester(self): + data = { + "name": "harvester", + "remote_url": "http://fake.com", + "harvesting_session_update_frequency": 60, + "refresh_harvestable_resources_update_frequency": 30, + "check_availability_frequency": 10, + "harvester_type_specific_configuration": "{}", + "harvester_type": self.harvester_type, + "status": models.Harvester.STATUS_READY, + "default_owner": self.user.pk, + } + self.assertFalse(models.Harvester.objects.filter(name=data["name"]).exists()) + response = self.client.post(reverse("admin:harvesting_harvester_add"), data) + print(f"response: {response.content}") + self.assertEqual(response.status_code, status.HTTP_302_FOUND) # response from admin + harvester = models.Harvester.objects.get(name=data["name"]) + self.assertEqual(harvester.name, data["name"]) + self.assertEqual(harvester.remote_url, data["remote_url"]) + self.assertEqual(harvester.status, models.Harvester.STATUS_READY) + + def test_update_harvester_availability(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + model_admin = admin.HarvesterAdmin(model=mock_harvester, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.update_harvester_availability(None, [mock_harvester]) + mock_harvester.update_availability.assert_called() + + def test_initiate_update_harvestable_resources_initiates_when_harvester_is_available(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + mock_harvester.update_availability.return_value = True + model_admin = admin.HarvesterAdmin(model=mock_harvester, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.initiate_update_harvestable_resources(None, [mock_harvester]) + mock_harvester.update_availability.assert_called() + mock_harvester.initiate_update_harvestable_resources.assert_called() + + def test_initiate_update_harvestable_resources_skips_when_harvester_not_available(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + mock_harvester.update_availability.return_value = False + model_admin = admin.HarvesterAdmin(model=mock_harvester, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.initiate_update_harvestable_resources(None, [mock_harvester]) + mock_harvester.update_availability.assert_called() + mock_harvester.initiate_update_harvestable_resources.assert_not_called() + + def test_initiate_perform_harvesting_initiates_when_harvester_is_available(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + mock_harvester.update_availability.return_value = True + model_admin = admin.HarvesterAdmin(model=mock_harvester, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.initiate_perform_harvesting(None, [mock_harvester]) + mock_harvester.update_availability.assert_called() + mock_harvester.initiate_perform_harvesting.assert_called() + + def test_initiate_perform_harvesting_skips_when_harvester_not_available(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + mock_harvester.update_availability.return_value = False + model_admin = admin.HarvesterAdmin(model=mock_harvester, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.initiate_perform_harvesting(None, [mock_harvester]) + mock_harvester.update_availability.assert_called() + mock_harvester.initiate_perform_harvesting.assert_not_called() + + def test_reset_harvester_status(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + mock_harvester.status = models.Harvester.STATUS_PERFORMING_HARVESTING + model_admin = admin.HarvesterAdmin(model=mock_harvester, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.reset_harvester_status(None, [mock_harvester]) + self.assertEqual(mock_harvester.status, models.Harvester.STATUS_READY) + mock_harvester.save.assert_called() + + +class AsynchronousHarvestingSessionAdminTestCase(GeoNodeBaseTestSupport): + def test_django_admin_does_not_allow_creating_new_session(self): + data = {"session_type": "harvesting", "harvester": 1000, "status": "pending"} + response = self.client.post(reverse("admin:harvesting_harvester_add"), data, follow=True) + print(response.redirect_chain) + self.assertRedirects(response, "/en-us/admin/login/?next=/en-us/admin/harvesting/harvester/add/") + + +class HarvestableResourceAdminTestCase(GeoNodeBaseTestSupport): + def test_initiate_harvest_selected_resources_initiates_when_harvester_is_available(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + mock_harvester.update_availability.return_value = True + fake_resource_id = 1000 + mock_resource = mock.MagicMock(spec=models.HarvestableResource, harvester=mock_harvester, id=fake_resource_id) + model_admin = admin.HarvestableResourceAdmin(model=mock_resource, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.initiate_harvest_selected_resources(None, [mock_resource]) + mock_harvester.update_availability.assert_called() + mock_harvester.initiate_perform_harvesting.assert_called_with([fake_resource_id]) + + def test_initiate_harvest_selected_resources_skips_when_harvester_not_available(self): + mock_harvester_model = mock.MagicMock(spec=models.Harvester) + mock_harvester = mock_harvester_model.return_value + mock_harvester.update_availability.return_value = False + fake_resource_id = 1000 + mock_resource = mock.MagicMock(spec=models.HarvestableResource, harvester=mock_harvester, id=fake_resource_id) + model_admin = admin.HarvestableResourceAdmin(model=mock_resource, admin_site=AdminSite()) + with mock.patch.object(model_admin, "message_user"): + model_admin.initiate_harvest_selected_resources(None, [mock_resource]) + mock_harvester.update_availability.assert_called() + mock_harvester.initiate_perform_harvesting.assert_not_called() diff --git a/geonode/harvesting/tests/test_api_serializers.py b/geonode/harvesting/tests/test_api_serializers.py new file mode 100644 index 0000000..bde6a92 --- /dev/null +++ b/geonode/harvesting/tests/test_api_serializers.py @@ -0,0 +1,298 @@ +from unittest import mock +from urllib.parse import urlparse + +from django.contrib.auth import get_user_model +from django.utils.timezone import now +from rest_framework.exceptions import ValidationError +from rest_framework.test import ( + APIRequestFactory, +) + +from geonode.tests.base import GeoNodeBaseTestSupport + +from .. import models +from ..api import serializers + + +_REQUEST_FACTORY = APIRequestFactory() + + +class BriefHarvesterSerializerTestCase(GeoNodeBaseTestSupport): + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" + + @classmethod + def setUpTestData(cls): + cls.harvester = models.Harvester.objects.create( + remote_url=cls.remote_url, name=cls.name, default_owner=cls.user, harvester_type=cls.harvester_type + ) + + def test_serializer_is_able_to_serialize_model_instance(self): + api_endpoint = "/api/v2/harvesters/" + request = _REQUEST_FACTORY.get(api_endpoint) + serializer = serializers.BriefHarvesterSerializer(self.harvester, context={"request": request}) + serialized = serializer.data + self.assertEqual(serialized["remote_url"], self.harvester.remote_url) + self.assertEqual(serialized["name"], self.harvester.name) + self.assertEqual(urlparse(serialized["links"]["self"]).path, f"{api_endpoint}{self.harvester.pk}/") + self.assertIsNotNone(serialized["links"]["harvestable_resources"]) + + +class HarvesterSerializerTestCase(GeoNodeBaseTestSupport): + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" + + @classmethod + def setUpTestData(cls): + cls.harvester = models.Harvester.objects.create( + remote_url=cls.remote_url, name=cls.name, default_owner=cls.user, harvester_type=cls.harvester_type + ) + + def test_serializer_is_able_to_serialize_model_instance(self): + api_endpoint = "/api/v2/harvesters/" + request = _REQUEST_FACTORY.get(api_endpoint) + serializer = serializers.BriefHarvesterSerializer(self.harvester, context={"request": request}) + serialized = serializer.data + self.assertEqual(serialized["remote_url"], self.harvester.remote_url) + self.assertEqual(serialized["name"], self.harvester.name) + self.assertEqual(urlparse(serialized["links"]["self"]).path, f"{api_endpoint}{self.harvester.pk}/") + self.assertIsNotNone(serialized["links"]["harvestable_resources"]) + + @mock.patch("geonode.harvesting.models.validate_worker_configuration") + def test_validate_also_validates_worker_specific_config(self, mock_validate_config): + input_data = { + "name": "phony", + "remote_url": "http://fake.com", + "user": 1, + "harvester_type": "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker", + "harvester_type_specific_configuration": {"something": "fake config"}, + } + + request = _REQUEST_FACTORY.post("/api/v2/harvesters/") + request.user = self.user + + serializer = serializers.HarvesterSerializer(data=input_data, context={"request": request}) + serializer.is_valid(raise_exception=True) + mock_validate_config.assert_called() + + def test_validate_does_not_allow_changing_status_and_worker_specific_config(self): + input_data = { + "name": "phony", + "remote_url": "http://fake.com", + "user": 1, + "harvester_type_specific_configuration": {"something": "fake config"}, + "status": "updating-harvestable-resources", + } + + request = _REQUEST_FACTORY.post("/api/v2/harvesters/") + request.user = self.user + + serializer = serializers.HarvesterSerializer(data=input_data, context={"request": request}) + with self.assertRaises(ValidationError): + serializer.is_valid(raise_exception=True) + + def test_create_does_not_allow_setting_custom_status(self): + input_data = { + "name": "phony", + "remote_url": "http://fake.com", + "user": 1, + "status": "updating-harvestable-resources", + } + + request = _REQUEST_FACTORY.post("/api/v2/harvesters/") + request.user = self.user + + serializer = serializers.HarvesterSerializer(data=input_data, context={"request": request}) + serializer.is_valid(raise_exception=True) + with self.assertRaises(ValidationError): + serializer.save() + + def test_create(self): + input_data = { + "name": "phony", + "remote_url": "http://fake.com", + "user": 1, + } + request = _REQUEST_FACTORY.post("/api/v2/harvesters/") + request.user = self.user + + serializer = serializers.HarvesterSerializer(data=input_data, context={"request": request}) + serializer.is_valid(raise_exception=True) + harvester = serializer.save() + self.assertEqual(harvester.name, input_data["name"]) + + def test_update_errors_out_if_current_status_is_not_ready(self): + request = _REQUEST_FACTORY.patch(f"/api/v2/harvesters/{self.harvester.pk}") + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + serializer = serializers.HarvesterSerializer( + self.harvester, + data={"status": "updating-harvestable-resources"}, + context={"request": request}, + partial=True, + ) + serializer.is_valid(raise_exception=True) + self.harvester.status = models.Harvester.STATUS_UPDATING_HARVESTABLE_RESOURCES + self.harvester.save() + with self.assertRaises(ValidationError): + serializer.save() + + def test_update_errors_out_when_client_tries_to_set_status_ready(self): + request = _REQUEST_FACTORY.patch(f"/api/v2/harvesters/{self.harvester.pk}") + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + serializer = serializers.HarvesterSerializer( + self.harvester, + data={"status": models.Harvester.STATUS_READY}, + context={"request": request}, + partial=True, + ) + serializer.is_valid(raise_exception=True) + with self.assertRaises(ValidationError): + serializer.save() + + @mock.patch("geonode.harvesting.api.serializers.tasks") + def test_update_calls_update_harvestable_resources_task(self, mock_tasks): + request = _REQUEST_FACTORY.patch(f"/api/v2/harvesters/{self.harvester.pk}") + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + serializer = serializers.HarvesterSerializer( + self.harvester, + data={"status": models.Harvester.STATUS_UPDATING_HARVESTABLE_RESOURCES}, + context={"request": request}, + partial=True, + ) + serializer.is_valid(raise_exception=True) + serializer.save() + mock_tasks.update_harvestable_resources.signature.assert_called() + mock_tasks.update_harvestable_resources.signature.return_value.apply_async.assert_called() + + @mock.patch("geonode.harvesting.api.serializers.tasks") + def test_update_calls_harvesting_dispatcher_task(self, mock_tasks): + request = _REQUEST_FACTORY.patch(f"/api/v2/harvesters/{self.harvester.pk}") + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + serializer = serializers.HarvesterSerializer( + self.harvester, + data={"status": models.Harvester.STATUS_PERFORMING_HARVESTING}, + context={"request": request}, + partial=True, + ) + serializer.is_valid(raise_exception=True) + serializer.save() + called_args = mock_tasks.harvesting_dispatcher.signature.call_args_list[0].kwargs["args"] + called_session_pk = called_args[0] + session = models.AsynchronousHarvestingSession.objects.get(pk=called_session_pk) + self.assertEqual(session.harvester.pk, self.harvester.pk) + mock_tasks.harvesting_dispatcher.signature.return_value.apply_async.assert_called() + + @mock.patch("geonode.harvesting.api.serializers.tasks") + def test_update_calls_update_harvester_availability_task(self, mock_tasks): + request = _REQUEST_FACTORY.patch(f"/api/v2/harvesters/{self.harvester.pk}") + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + serializer = serializers.HarvesterSerializer( + self.harvester, + data={"status": models.Harvester.STATUS_CHECKING_AVAILABILITY}, + context={"request": request}, + partial=True, + ) + serializer.is_valid(raise_exception=True) + serializer.save() + mock_tasks.check_harvester_available.signature.assert_called_with(args=(self.harvester.pk,)) + mock_tasks.check_harvester_available.signature.return_value.apply_async.assert_called() + + @mock.patch("geonode.harvesting.api.serializers.tasks") + def test_update_updates_harvestable_resources_whenever_worker_config_changes(self, mock_tasks): + request = _REQUEST_FACTORY.patch(f"/api/v2/harvesters/{self.harvester.pk}") + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + self.assertEqual(len(self.harvester.harvester_type_specific_configuration), 0) + serializer = serializers.HarvesterSerializer( + self.harvester, + data={"harvester_type_specific_configuration": {"harvest_datasets": False}}, + context={"request": request}, + partial=True, + ) + serializer.is_valid(raise_exception=True) + serializer.save() + mock_tasks.update_harvestable_resources.signature.assert_called_with(args=(self.harvester.pk,)) + mock_tasks.update_harvestable_resources.signature.return_value.apply_async.assert_called() + + +class BriefAsynchronousHarvestingSessionSerializerTestCase(GeoNodeBaseTestSupport): + harvester: models.Harvester + + @classmethod + def setUpTestData(cls): + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" + cls.harvester = models.Harvester.objects.create( + remote_url=remote_url, name=name, default_owner=user, harvester_type=harvester_type + ) + cls.harvesting_session = models.AsynchronousHarvestingSession.objects.create( + harvester=cls.harvester, session_type=models.AsynchronousHarvestingSession.TYPE_HARVESTING + ) + + def test_serializer_is_able_to_serialize_model_instance(self): + api_endpoint = "/api/v2/harvesting-sessions/" + request = _REQUEST_FACTORY.get(api_endpoint) + serializer = serializers.BriefAsynchronousHarvestingSessionSerializer( + self.harvesting_session, context={"request": request} + ) + serialized = serializer.data + self.assertIsNotNone(serialized["started"]) + + +class HarvestableResourceSerializerTestCase(GeoNodeBaseTestSupport): + unique_identifier = "some-identifier" + title = "something" + remote_resource_type = "documents" + default_should_be_harvested = False + + @classmethod + def setUpTestData(cls): + cls.harvester = models.Harvester.objects.create( + remote_url="test.com", + name="This is geonode harvester", + default_owner=get_user_model().objects.get(username="AnonymousUser"), + harvester_type="geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker", + ) + cls.harvestable_resource = models.HarvestableResource.objects.create( + unique_identifier=cls.unique_identifier, + title=cls.title, + harvester=cls.harvester, + should_be_harvested=cls.default_should_be_harvested, + remote_resource_type=cls.remote_resource_type, + last_refreshed=now(), + ) + + def test_serializer_is_able_to_serialize_model_instance(self): + api_endpoint = f"/api/v2/harvesters/{self.harvester.id}/harvestable-resources/" + request = _REQUEST_FACTORY.get(api_endpoint) + serializer = serializers.HarvestableResourceSerializer(self.harvestable_resource, context={"request": request}) + serialized = serializer.data + self.assertIsNotNone(serialized["unique_identifier"], self.unique_identifier) + self.assertIsNotNone(serialized["title"], self.title) + self.assertIsNotNone(serialized["remote_resource_type"], self.remote_resource_type) + self.assertIsNotNone(serialized["should_be_harvested"], self.default_should_be_harvested) + + def test_serializer_is_allowed_to_change_instance_should_be_harvested_attribute(self): + self.assertEqual(self.harvestable_resource.should_be_harvested, self.default_should_be_harvested) + api_endpoint = f"/api/v2/harvesters/{self.harvester.id}/harvestable-resources/" + request = _REQUEST_FACTORY.patch(api_endpoint) + serializer = serializers.HarvestableResourceSerializer( + data={ + "unique_identifier": self.unique_identifier, + "should_be_harvested": not self.default_should_be_harvested, + }, + context={ + "request": request, + "harvester": self.harvester, + }, + ) + serializer.is_valid(raise_exception=True) + print(f"validated_data: {serializer.validated_data}") + serializer.save() + self.harvestable_resource.refresh_from_db() + self.assertEqual(self.harvestable_resource.should_be_harvested, not self.default_should_be_harvested) diff --git a/geonode/harvesting/tests/test_api_views.py b/geonode/harvesting/tests/test_api_views.py new file mode 100644 index 0000000..c7bfac4 --- /dev/null +++ b/geonode/harvesting/tests/test_api_views.py @@ -0,0 +1,95 @@ +import datetime +from django.contrib.auth import get_user_model +from rest_framework import status +from geonode.tests.base import GeoNodeBaseTestSupport + +from .. import models + + +class HarvesterViewSetTestCase(GeoNodeBaseTestSupport): + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + + @classmethod + def setUpTestData(cls): + harvester1 = models.Harvester.objects.create( + remote_url="http://fake1.com", name="harvester1", default_owner=cls.user, harvester_type=cls.harvester_type + ) + harvester2 = models.Harvester.objects.create( + remote_url="http://fake2.com", name="harvester2", default_owner=cls.user, harvester_type=cls.harvester_type + ) + cls.harvesters = [harvester1, harvester2] + + harvester_resource1 = models.HarvestableResource.objects.create( + unique_identifier="resource_1", + title="resource 1", + harvester=harvester1, + last_refreshed=datetime.datetime.now(), + ) + harvester_resource2 = models.HarvestableResource.objects.create( + unique_identifier="resource_2", + title="resource 2", + harvester=harvester2, + last_refreshed=datetime.datetime.now(), + ) + cls.resources = { + harvester1.id: harvester_resource1, + harvester2.id: harvester_resource2, + } + + session1 = models.AsynchronousHarvestingSession.objects.create( + harvester=harvester1, session_type=models.AsynchronousHarvestingSession.TYPE_HARVESTING, records_done=10 + ) + session2 = models.AsynchronousHarvestingSession.objects.create( + harvester=harvester2, session_type=models.AsynchronousHarvestingSession.TYPE_HARVESTING, records_done=5 + ) + cls.sessions = [session1, session2] + + def test_get_harvester_list(self): + response = self.client.get("/api/v2/harvesters/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["total"], len(self.harvesters)) + for index, harvester in enumerate(self.harvesters): + self.assertIn(self.harvesters[index].pk, [i["id"] for i in response.data["harvesters"]]) + + # self.assertEqual(response.data["harvesters"][index]["id"], self.harvesters[index].pk) + # self.assertEqual(response.data["harvesters"][index]["name"], self.harvesters[index].name) + + def test_post_harvester_list_non_admin(self): + response = self.client.post("/api/v2/harvesters/", {}) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_get_harvester_detail_for_non_admin(self): + self.client.logout() + for index, harvester in enumerate(self.harvesters): + response = self.client.get(f"/api/v2/harvesters/{harvester.id}/") + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_get_harvester_detail_for_admin(self): + self.client.login(username="admin", password="admin") + for index, harvester in enumerate(self.harvesters): + response = self.client.get(f"/api/v2/harvesters/{harvester.id}/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["harvester"]["id"], harvester.pk) + self.assertEqual(response.data["harvester"]["name"], harvester.name) + + def test_get_harvester_resources(self): + for index, harvester in enumerate(self.harvesters): + response = self.client.get(f"/api/v2/harvesters/{harvester.id}/harvestable-resources/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + response.data["harvestable_resources"][0]["unique_identifier"], + self.resources[harvester.id].unique_identifier, + ) + self.assertEqual(response.data["harvestable_resources"][0]["title"], self.resources[harvester.id].title) + + def test_get_harvester_sessions(self): + response = self.client.get("/api/v2/harvesting-sessions/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["total"], len(self.sessions)) + for index, harvester in enumerate(self.sessions): + self.assertEqual(response.data["asynchronous_harvesting_sessions"][index]["id"], self.sessions[index].pk) + self.assertEqual( + response.data["asynchronous_harvesting_sessions"][index]["records_done"], + self.sessions[index].records_done, + ) diff --git a/geonode/harvesting/tests/test_config.py b/geonode/harvesting/tests/test_config.py new file mode 100644 index 0000000..f10b281 --- /dev/null +++ b/geonode/harvesting/tests/test_config.py @@ -0,0 +1,48 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.test import override_settings + +from geonode.harvesting import config +from geonode.tests.base import GeoNodeBaseSimpleTestSupport + + +class ConfigTestCase(GeoNodeBaseSimpleTestSupport): + @override_settings(HARVESTER_CLASSES=[]) + def test_default_config_harvester_classes(self): + self.assertEqual(config.get_setting("HARVESTER_CLASSES"), config._DEFAULT_HARVESTERS) + + def test_custom_harvester_classes(self): + phony_class_paths = [ + "fake_harvester1", + "fake_harvester2", + ] + with self.settings(HARVESTER_CLASSES=phony_class_paths): + self.assertEqual(config.get_setting("HARVESTER_CLASSES"), config._DEFAULT_HARVESTERS + phony_class_paths) + + def test_harvester_classes_dont_repeat(self): + phony_class_paths = [ + "fake_harvester1", + "fake_harvester2", + ] + repeated_paths = [ + "fake_harvester1", + ] + with self.settings(HARVESTER_CLASSES=phony_class_paths + repeated_paths): + self.assertEqual(config.get_setting("HARVESTER_CLASSES"), config._DEFAULT_HARVESTERS + phony_class_paths) diff --git a/geonode/harvesting/tests/test_harvester_worker_arcgis.py b/geonode/harvesting/tests/test_harvester_worker_arcgis.py new file mode 100644 index 0000000..df973f0 --- /dev/null +++ b/geonode/harvesting/tests/test_harvester_worker_arcgis.py @@ -0,0 +1,95 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from unittest import mock + +from geonode.harvesting.harvesters import arcgis +from geonode.tests.base import GeoNodeBaseSimpleTestSupport + + +class ArcgisModuleTestCase(GeoNodeBaseSimpleTestSupport): + def test_parse_spatial_extent_with_latest_wkid(self): + raw_extent = { + "xmin": 10, + "ymin": 5, + "xmax": 20, + "ymax": 45, + "spatialReference": { + "wkid": 102100, + "latestWkid": 3857, + }, + } + epsg_code, polygon = arcgis._parse_spatial_extent(raw_extent) + self.assertEqual(epsg_code, "EPSG:3857") + self.assertEqual(polygon.wkt, "POLYGON ((10 5, 10 45, 20 45, 20 5, 10 5))") + + def test_parse_spatial_extent_without_latest_wkid(self): + raw_extent = { + "xmin": 10, + "ymin": 5, + "xmax": 20, + "ymax": 45, + "spatialReference": { + "wkid": 102100, + }, + } + epsg_code, polygon = arcgis._parse_spatial_extent(raw_extent) + self.assertEqual(epsg_code, "EPSG:102100") + self.assertEqual(polygon.wkt, "POLYGON ((10 5, 10 45, 20 45, 20 5, 10 5))") + + def test_parse_remote_url(self): + fixtures = [ + ("https://fake/rest/services/myservice/MapServer", "https://fake/rest/services", "myservice", "MapServer"), + ( + "https://fake/rest/services/myservice/MapServer/Query", + "https://fake/rest/services", + "myservice", + "MapServer", + ), + ("https://fake/rest/services", "https://fake/rest/services", None, None), + ( + "https://fake/rest/services/myservice/ImageServer", + "https://fake/rest/services", + "myservice", + "ImageServer", + ), + ] + for url, expected_cat_url, expected_service_name, expected_service_type in fixtures: + cat_url, service_name, service_type = arcgis.parse_remote_url(url) + self.assertEqual(cat_url, expected_cat_url) + self.assertEqual(service_name, expected_service_name) + self.assertEqual(service_type, expected_service_type) + + @mock.patch("geonode.harvesting.harvesters.arcgis.arcrest") + def test_get_resource_extractor(self, mock_arcrest): + fixtures = [ + ( + "http://somewhere/rest/services/fakeservice1/MapServer/1", + mock_arcrest.MapService, + arcgis.ArcgisMapServiceResourceExtractor, + ), + ( + "http://somewhere/rest/services/fakeservice1/ImageServer/1", + mock_arcrest.ImageService, + arcgis.ArcgisImageServiceResourceExtractor, + ), + ] + for identifier, mock_class, extractor_class in fixtures: + result = arcgis.get_resource_extractor(identifier) + mock_class.assert_called_with(identifier) + self.assertIsInstance(result, extractor_class) diff --git a/geonode/harvesting/tests/test_harvester_worker_geonode_legacy.py b/geonode/harvesting/tests/test_harvester_worker_geonode_legacy.py new file mode 100644 index 0000000..d7bded8 --- /dev/null +++ b/geonode/harvesting/tests/test_harvester_worker_geonode_legacy.py @@ -0,0 +1,90 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from unittest import mock + +from django.utils import timezone + +from geonode.harvesting.harvesters import geonodeharvester +from geonode.tests.base import GeoNodeBaseSimpleTestSupport + +from .. import models + + +class GeoNodeHarvesterWorkerTestCase(GeoNodeBaseSimpleTestSupport): + def test_base_api_url(self): + """Base API URL does not have an extra slash, regardless of whether the original URL has it or not""" + base_remote_urls = [ + ("http://fake-url1", "http://fake-url1/api"), + ("http://fake-url2/", "http://fake-url2/api"), + ] + harvester_id = "fake-id" + for base_url, expected in base_remote_urls: + worker = geonodeharvester.GeonodeLegacyHarvester(base_url, harvester_id) + self.assertEqual(worker.base_api_url, expected) + + def test_that_copying_remote_resources_is_allowed(self): + worker = geonodeharvester.GeonodeLegacyHarvester("http://fake-url2/", "fake-id") + self.assertTrue(worker.allows_copying_resources) + + def test_creation_from_harvester(self): + now = timezone.now() + keywords = ["keyword1", "keyword2"] + categories = ["category1", "category2"] + combinations = [ + { + "harvest_documents": True, + "harvest_datasets": True, + "copy_datasets": True, + "copy_documents": True, + "resource_title_filter": "something", + "start_date_filter": now, + "end_date_filter": now, + "keywords_filter": keywords, + "categories_filter": categories, + }, + ] + for param_combination in combinations: + harvester = models.Harvester( + name="fake1", + harvester_type="geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester", + harvester_type_specific_configuration=param_combination, + ) + harvester.get_harvester_worker() + + @mock.patch("geonode.harvesting.harvesters.geonodeharvester.requests.Session") + def test_check_availability_works_when_response_includes_layers_object(self, mock_requests_session): + mock_response = mock.MagicMock() + mock_response.json.return_value = {"layers": []} + mock_requests_session.return_value.get.return_value = mock_response + + base_url = "http://fake-url1" + worker = geonodeharvester.GeonodeLegacyHarvester(base_url, harvester_id=None) + result = worker.check_availability() + self.assertEqual(result, True) + + @mock.patch("geonode.harvesting.harvesters.geonodeharvester.requests.Session") + def test_check_availability_fails_when_response_does_not_include_layers_object(self, mock_requests_session): + mock_response = mock.MagicMock() + mock_response.json.return_value = {} + mock_requests_session.return_value.get.return_value = mock_response + + base_url = "http://fake-url2" + worker = geonodeharvester.GeonodeLegacyHarvester(base_url, harvester_id=None) + result = worker.check_availability() + self.assertEqual(result, False) diff --git a/geonode/harvesting/tests/test_harvester_worker_wms.py b/geonode/harvesting/tests/test_harvester_worker_wms.py new file mode 100644 index 0000000..705ed4b --- /dev/null +++ b/geonode/harvesting/tests/test_harvester_worker_wms.py @@ -0,0 +1,31 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.harvesting.harvesters import wms +from geonode.tests.base import GeoNodeBaseSimpleTestSupport + + +class WmsModuleTestCase(GeoNodeBaseSimpleTestSupport): + def test_get_nsmap(self): + fixtures = [ + ({None: "ns1uri", "ns2": "ns2uri"}, {"wms": "ns1uri", "ns2": "ns2uri"}), + ({"ns1": "ns1uri", "ns2": "ns2uri"}, {"ns1": "ns1uri", "ns2": "ns2uri"}), + ] + for original, expected in fixtures: + result = wms._get_nsmap(original) + self.assertEqual(result, expected) diff --git a/geonode/harvesting/tests/test_integrations.py b/geonode/harvesting/tests/test_integrations.py new file mode 100644 index 0000000..37d3cb7 --- /dev/null +++ b/geonode/harvesting/tests/test_integrations.py @@ -0,0 +1,100 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import uuid +import datetime as dt +from unittest import mock + +from django.test.utils import override_settings +from django.contrib.auth import get_user_model +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.layers.models import Dataset + +from geonode.harvesting.resourcedescriptor import ( + RecordDescriptionContact, + RecordDistribution, + RecordIdentification, + RecordDescription, +) +from geonode.harvesting.models import Harvester, HarvestableResource, AsynchronousHarvestingSession +from geonode.harvesting.harvesters.base import HarvestedResourceInfo +from geonode.harvesting.harvesters.geonodeharvester import GeonodeUnifiedHarvesterWorker + + +class HarvesterIntegrationsTestCase(GeoNodeBaseTestSupport): + unique_identifier = "id" + title = "Test" + remote_url = "test.com" + name = "This is a geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + + @override_settings(ASYNC_SIGNALS=False) + def setUp(self): + super().setUp() + + self.record_description_contact = mock.MagicMock(RecordDescriptionContact) + self.record_distribution = mock.MagicMock(RecordDistribution) + self.record_distribution.thumbnail_url.return_value = "thumb.png" + self.record_distribution.wms_url.return_value = "http://test.com/geoserver" + self.record_identification = RecordIdentification(name=self.name, title=self.title, other_keywords=[]) + + self.harvester = Harvester.objects.create( + remote_url=self.remote_url, name=self.name, default_owner=self.user, harvester_type=self.harvester_type + ) + self.harvestable_resource = HarvestableResource.objects.create( + unique_identifier=self.unique_identifier, + title=self.title, + harvester=self.harvester, + last_refreshed=dt.datetime.now(), + remote_resource_type="dataset", + ) + + self.session = AsynchronousHarvestingSession.objects.create( + harvester=self.harvester, session_type=AsynchronousHarvestingSession.TYPE_HARVESTING + ) + + self.record_description = RecordDescription( + uuid=uuid.uuid4(), + author=self.record_description_contact, + date_stamp=dt.datetime.now(), + distribution=self.record_distribution, + identification=self.record_identification, + point_of_contact=self.record_description_contact, + reference_systems=["EPSG:3857"], + additional_parameters={ + "alternate": "geonode:test", + "workspace": "geonode", + "subtype": "vector", + "resource_type": "", + }, + ) + + self.harvested_info = HarvestedResourceInfo( + resource_descriptor=self.record_description, copied_resources=[], additional_information=None + ) + + @mock.patch( + "geonode.harvesting.harvesters.geonodeharvester.GeonodeCurrentHarvester.check_availability", + mock.Mock(return_value=True), + ) + def test_remote_subtype_is_set_for_harvested_dataset(self): + worker = GeonodeUnifiedHarvesterWorker(remote_url=self.remote_url, harvester_id=self.harvester.id) + worker.update_geonode_resource(self.harvested_info, self.harvestable_resource) + dataset = Dataset.objects.get(alternate="geonode:test") + self.assertEqual(dataset.subtype, "remote") diff --git a/geonode/harvesting/tests/test_models.py b/geonode/harvesting/tests/test_models.py new file mode 100644 index 0000000..cd7af3c --- /dev/null +++ b/geonode/harvesting/tests/test_models.py @@ -0,0 +1,171 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import datetime as dt +from unittest import mock + +from django.contrib.auth import get_user_model +from django.test import SimpleTestCase +from geonode.tests.base import GeoNodeBaseTestSupport + +from .. import models + + +class HarvesterTestCase(GeoNodeBaseTestSupport): + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + + def setUp(self): + super().setUp() + self.harvester = models.Harvester.objects.create( + remote_url=self.remote_url, name=self.name, default_owner=self.user, harvester_type=self.harvester_type + ) + + def test_get_worker_works(self): + worker = self.harvester.get_harvester_worker() + self.assertEqual(worker.remote_url, self.remote_url) + + @mock.patch("geonode.harvesting.models.jsonschema") + @mock.patch("geonode.harvesting.models.import_string") + def test_validate_worker_configuration(self, mock_import_string, mock_jsonschema): + extra_config_schema = "fake_config_schema" + mock_worker_class = mock.MagicMock() + mock_worker_class.get_extra_config_schema.return_value = extra_config_schema + mock_import_string.return_value = mock_worker_class + + harvester_type = "fake_harvester_type" + configuration = {"fake_key": "fake_configuration"} + models.validate_worker_configuration(harvester_type, configuration) + + mock_import_string.assert_called_with(harvester_type) + mock_worker_class.get_extra_config_schema.assert_called() + mock_jsonschema.validate.assert_called_with(configuration, extra_config_schema) + + @mock.patch("geonode.harvesting.models.timezone") + def test_get_next_check_availability_dispatch_time(self, mock_timezone): + fixtures = [ + ("2020-01-01T00:00:00", "2020-01-01T00:05:00", 10, "2020-01-01T00:10:00"), + ("2020-01-01T00:00:00", "2020-01-01T00:11:00", 10, "2020-01-01T00:11:00"), + ] + for last_checked, now, frequency, expected in fixtures: + mock_timezone.now.return_value = dt.datetime.fromisoformat(now).replace(tzinfo=dt.timezone.utc) + harvester = models.Harvester(check_availability_frequency=frequency) + harvester.last_checked_availability = dt.datetime.fromisoformat(last_checked).replace( + tzinfo=dt.timezone.utc + ) + result = harvester.get_next_check_availability_dispatch_time() + expected_result = dt.datetime.fromisoformat(expected).replace(tzinfo=dt.timezone.utc) + self.assertEqual(result, expected_result) + + @mock.patch("geonode.harvesting.models.timezone") + def test_get_next_dispatch_time(self, mock_timezone): + fixtures = [ + ("2020-01-01T00:00:00", "2020-01-01T00:05:00", 10, "2020-01-01T00:10:00"), + ("2020-01-01T00:00:00", "2020-01-01T00:11:00", 10, "2020-01-01T00:11:00"), + ] + for last_check, now, frequency, expected in fixtures: + mock_timezone.now.return_value = dt.datetime.fromisoformat(now).replace(tzinfo=dt.timezone.utc) + with mock.patch.object( + models.Harvester, "latest_refresh_session", new_callable=mock.PropertyMock + ) as mock_latest_refresh_session: + mock_latest_refresh_session.return_value = mock.MagicMock( + started=dt.datetime.fromisoformat(last_check).replace(tzinfo=dt.timezone.utc) + ) + harvester = models.Harvester( + harvesting_session_update_frequency=10, + refresh_harvestable_resources_update_frequency=10, + ) + result = harvester._get_next_dispatch_time( + models.AsynchronousHarvestingSession.TYPE_DISCOVER_HARVESTABLE_RESOURCES + ) + expected_result = dt.datetime.fromisoformat(expected).replace(tzinfo=dt.timezone.utc) + self.assertEqual(result, expected_result) + + +class AsynchronousHarvestingSessionTestCase(GeoNodeBaseTestSupport): + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + + def setUp(self): + super().setUp() + self.harvester = models.Harvester.objects.create( + remote_url=self.remote_url, name=self.name, default_owner=self.user, harvester_type=self.harvester_type + ) + self.harvesting_session = models.AsynchronousHarvestingSession.objects.create( + harvester=self.harvester, session_type=models.AsynchronousHarvestingSession.TYPE_HARVESTING + ) + + def test_check_attributes(self): + """ + Test attributes of harvester_session after created. + """ + self.assertIsNotNone(self.harvesting_session.pk) + self.assertEqual(self.harvesting_session.harvester, self.harvester) + self.assertEqual(self.harvesting_session.records_done, 0) + + +class HarvestableResourceTestCase(GeoNodeBaseTestSupport): + unique_identifier = "id" + title = "Test" + remote_url = "test.com" + name = "This is geonode harvester" + user = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + + def setUp(self): + super().setUp() + self.harvester = models.Harvester.objects.create( + remote_url=self.remote_url, name=self.name, default_owner=self.user, harvester_type=self.harvester_type + ) + self.harvestable_resource = models.HarvestableResource.objects.create( + unique_identifier=self.unique_identifier, + title=self.title, + harvester=self.harvester, + last_refreshed=dt.datetime.now(), + ) + + def test_check_attributes(self): + self.assertIsNotNone(self.harvestable_resource.pk) + self.assertEqual(self.harvestable_resource.harvester, self.harvester) + self.assertEqual(self.harvestable_resource.title, self.title) + self.assertEqual(self.harvestable_resource.unique_identifier, self.unique_identifier) + self.assertFalse(self.harvestable_resource.should_be_harvested) + self.assertEqual(self.harvestable_resource.status, models.HarvestableResource.STATUS_READY) + + +class WorkerConfigValidationTestCase(SimpleTestCase): + @mock.patch("geonode.harvesting.models.jsonschema") + @mock.patch("geonode.harvesting.models.import_string") + def test_validate_worker_configuration(self, mock_import_string, mock_jsonschema): + extra_config_schema = "fake_config_schema" + mock_worker_class = mock.MagicMock() + mock_worker_class.get_extra_config_schema.return_value = extra_config_schema + mock_import_string.return_value = mock_worker_class + + harvester_type = "fake_harvester_type" + configuration = {"somekey": "fake_configuration"} + models.validate_worker_configuration(harvester_type, configuration) + + mock_import_string.assert_called_with(harvester_type) + mock_worker_class.get_extra_config_schema.assert_called() + mock_jsonschema.validate.assert_called_with(configuration, extra_config_schema) diff --git a/geonode/harvesting/tests/test_tasks.py b/geonode/harvesting/tests/test_tasks.py new file mode 100644 index 0000000..e57f6d2 --- /dev/null +++ b/geonode/harvesting/tests/test_tasks.py @@ -0,0 +1,350 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from unittest import mock + +from django.contrib.auth import get_user_model +from django.test import override_settings +from django.utils.timezone import now +from geonode.tests.base import GeoNodeBaseTestSupport + +from .. import ( + models, + tasks, +) + + +class TasksTestCase(GeoNodeBaseTestSupport): + harvester: models.Harvester + harvester_remote_url = "fake url" + harvester_name = "harvester1" + harvester_owner = get_user_model().objects.get(username="AnonymousUser") + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + + @classmethod + def setUpTestData(cls): + cls.harvester = models.Harvester.objects.create( + remote_url=cls.harvester_remote_url, + name=cls.harvester_name, + status=models.Harvester.STATUS_UPDATING_HARVESTABLE_RESOURCES, + default_owner=cls.harvester_owner, + harvester_type=cls.harvester_type, + ) + cls.harvesting_session = models.AsynchronousHarvestingSession.objects.create( + harvester=cls.harvester, session_type=models.AsynchronousHarvestingSession.TYPE_HARVESTING + ) + for index in range(3): + models.HarvestableResource.objects.create( + unique_identifier=f"fake-identifier-{index}", + title=f"fake-title-{index}", + harvester=cls.harvester, + remote_resource_type="fake-remote-resource-type", + last_refreshed=now(), + ) + + @mock.patch("geonode.harvesting.tasks.update_asynchronous_session") + def test_harvest_resource_updates_geonode_when_remote_resource_exists(self, mock_update_asynchronous_session): + """Test that `worker.get_resource()` is called by the `_harvest_resource()` task and that the related workflow is called too. + + Verify that `worker.get_resource()` is always called. Then verify that if the result of `worker.get_resource()` is + not `None`, the `worker.update_geonode_resource()` is called and `worker.update_harvesting_session()` is called too. + + """ + + harvestable_resource_id = "fake id" + with mock.patch("geonode.harvesting.tasks.models") as mock_models: + mock_worker = mock.MagicMock() + mock_worker.get_resource.return_value = "fake_gotten_resource" + mock_worker.should_copy_resource.return_value = False + mock_harvestable_resource = mock.MagicMock(models.HarvestableResource) + mock_harvestable_resource.harvester.get_harvester_worker.return_value = mock_worker + mock_models.HarvestableResource.objects.get.return_value = mock_harvestable_resource + + tasks._harvest_resource(harvestable_resource_id, self.harvesting_session.id) + + mock_models.HarvestableResource.objects.get.assert_called_with(pk=harvestable_resource_id) + mock_worker.get_resource.assert_called() + mock_worker.update_geonode_resource.assert_called() + mock_update_asynchronous_session.assert_called() + + def test_harvest_resource_does_not_update_geonode_when_remote_resource_does_not_exist(self): + """Test that the worker does not try to update existing GeoNode resources when the remote resource cannot be harvested.""" + + harvestable_resource_id = "fake id" + with mock.patch("geonode.harvesting.tasks.models") as mock_models: + mock_worker = mock.MagicMock() + mock_worker.get_resource.return_value = None # this means the remote resource was not harvested + mock_worker.should_copy_resource.return_value = False + mock_harvestable_resource = mock.MagicMock(models.HarvestableResource) + mock_harvestable_resource.harvester.get_harvester_worker.return_value = mock_worker + mock_models.HarvestableResource.objects.get.return_value = mock_harvestable_resource + + tasks._harvest_resource(harvestable_resource_id, self.harvesting_session.id) + + mock_models.HarvestableResource.objects.get.assert_called_with(pk=harvestable_resource_id) + mock_worker.get_resource.assert_called() + mock_worker.update_geonode_resource.assert_not_called() + + def test_finish_harvesting_updates_harvester_status(self): + tasks._finish_harvesting(self.harvesting_session.id) + self.harvester.refresh_from_db() + self.harvesting_session.refresh_from_db() + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + self.assertIsNotNone(self.harvesting_session.ended) + + def test_handle_harvesting_error_cleans_up_harvest_execution(self): + tasks._handle_harvesting_error( + None, harvester_id=self.harvester.id, harvesting_session_id=self.harvesting_session.id + ) + self.harvester.refresh_from_db() + self.harvesting_session.refresh_from_db() + self.assertEqual(self.harvester.status, models.Harvester.STATUS_READY) + self.assertIsNotNone(self.harvesting_session.ended) + + @mock.patch("geonode.harvesting.tasks.models.Harvester") + def test_check_harvester_available(self, mock_harvester_model): + mock_harvester = mock.MagicMock(spec=models.Harvester).return_value + mock_harvester_model.objects.get.return_value = mock_harvester + tasks.check_harvester_available(1000) + mock_harvester.update_availability.assert_called() + + @mock.patch("geonode.harvesting.tasks._handle_harvestable_resources_update_error") + @mock.patch("geonode.harvesting.tasks._finish_harvestable_resources_update") + @mock.patch("geonode.harvesting.tasks._update_harvestable_resources_batch") + @mock.patch("geonode.harvesting.tasks.chord") + @mock.patch("geonode.harvesting.tasks.models") + def test_update_harvestable_resources_sends_batched_requests( + self, mock_models, mock_chord, mock_batch, mock_finalizer, mock_error_handler + ): + """Verify that the `update_harvestable_resources` task creates a celery chord with the batched task, a finalizer and an error handler.""" + mock_worker = mock.MagicMock() + mock_worker.get_num_available_resources.return_value = 1 + mock_harvester = mock.MagicMock(models.Harvester) + mock_models.Harvester.objects.get.return_value = mock_harvester + mock_harvester.get_harvester_worker.return_value = mock_worker + + tasks.update_harvestable_resources("fake harvester id") + + mock_batch.signature.assert_called() + mock_finalizer.signature.assert_called() + mock_error_handler.signature.assert_called() + mock_chord.assert_called() + mock_chord.return_value.apply_async.assert_called() + + def test_harvesting_scheduler(self): + mock_harvester = mock.MagicMock(spec=models.Harvester).return_value + mock_harvester.scheduling_enabled = True + mock_harvester.is_harvestable_resources_refresh_due.return_value = True + mock_harvester.is_harvesting_due.return_value = True + with mock.patch("geonode.harvesting.tasks.models.Harvester.objects") as mock_qs: + mock_qs.all.return_value = [mock_harvester] + tasks.harvesting_scheduler() + mock_harvester.is_availability_check_due.assert_called() + mock_harvester.is_harvestable_resources_refresh_due.assert_called() + mock_harvester.initiate_update_harvestable_resources.assert_called() + mock_harvester.is_harvesting_due.assert_called() + mock_harvester.initiate_perform_harvesting.assert_called() + + @mock.patch("geonode.harvesting.tasks.transaction.on_commit") + @mock.patch("geonode.harvesting.tasks.models") + def test_harvest_resources_with_chunks( + self, + mock_models, + mock_on_commit, + ): + mock_session = mock.MagicMock() + mock_session.status = mock_session.STATUS_ON_GOING + mock_session.harvester.update_availability.return_value = True + mock_models.AsynchronousHarvestingSession.objects.get.return_value = mock_session + + harvestable_resource_ids = list(range(5)) + + with mock.patch("geonode.harvesting.tasks.queue_next_chunk_batch.apply_async") as mock_apply_async: + with override_settings(CHUNK_SIZE=2, MAX_PARALLEL_QUEUE_CHUNKS=2): + tasks.harvest_resources(harvestable_resource_ids, self.harvesting_session.id) + + # Simulate transaction.on_commit callback being run + assert mock_on_commit.called + callback = mock_on_commit.call_args[0][0] + callback() + + # Now apply_async should have been called + mock_apply_async.assert_called_once() + _, kwargs = mock_apply_async.call_args + + expected_expires = 5 * 20 + 300 # 5 resources * 20 + 300 buffer = 400 + expected_time_limit = 2 * 2 * 20 + 300 # CHUNK_SIZE * MAX_PARALLEL_QUEUE_CHUNKS * 20 + 300 = 380 + + self.assertEqual(kwargs["expires"], expected_expires) + self.assertEqual(kwargs["time_limit"], expected_time_limit) + + @mock.patch("geonode.harvesting.tasks.transaction.on_commit") + @mock.patch("geonode.harvesting.tasks.models") + def test_harvest_resources_without_chunks(self, mock_models, mock_on_commit): + mock_session = mock.MagicMock() + mock_session.status = mock_session.STATUS_ON_GOING + mock_session.harvester.update_availability.return_value = True + mock_models.AsynchronousHarvestingSession.objects.get.return_value = mock_session + + harvestable_resource_ids = list(range(5)) + + with mock.patch("geonode.harvesting.tasks.chord") as mock_chord: + mock_workflow = mock.MagicMock() + mock_chord.return_value = mock_workflow + + with override_settings(CHUNK_SIZE=100, MAX_PARALLEL_QUEUE_CHUNKS=2): + tasks.harvest_resources(harvestable_resource_ids, self.harvesting_session.id) + + # Check that transaction.on_commit was called + self.assertTrue(mock_on_commit.called) + callback = mock_on_commit.call_args[0][0] + + # Simulate the transaction commit + callback() + + # Check that chord was built with correct number of subtasks + self.assertTrue(mock_chord.called) + subtasks = mock_chord.call_args[0][0] # This is the list of resource tasks + self.assertEqual(len(subtasks), len(harvestable_resource_ids)) + + # Check that apply_async was called on the workflow + mock_workflow.apply_async.assert_called_once() + + @mock.patch("geonode.harvesting.tasks.logger") + @mock.patch("geonode.harvesting.tasks.models") + def test_harvest_resources_aborted_session(self, mock_models, mock_logger): + mock_session = mock.MagicMock() + mock_session.status = mock_session.STATUS_ABORTED + mock_models.AsynchronousHarvestingSession.objects.get.return_value = mock_session + + tasks.harvest_resources([1, 2, 3], self.harvesting_session.id) + + mock_logger.debug.assert_any_call("Session has been aborted, skipping...") + # Ensure nothing else happened + mock_session.harvester.update_availability.assert_not_called() + + @mock.patch("geonode.harvesting.tasks.finish_asynchronous_session") + @mock.patch("geonode.harvesting.tasks.logger") + @mock.patch("geonode.harvesting.tasks.models") + def test_harvest_resources_no_ids(self, mock_models, mock_logger, mock_finish): + mock_session = mock.MagicMock() + mock_session.status = mock_session.STATUS_ON_GOING + mock_models.AsynchronousHarvestingSession.objects.get.return_value = mock_session + + tasks.harvest_resources([], self.harvesting_session.id) + + mock_logger.debug.assert_any_call("harvest_resources - Nothing to do...") + mock_finish.assert_called_once_with( + self.harvesting_session.id, + mock_models.AsynchronousHarvestingSession.STATUS_FINISHED_ALL_OK, + final_details="No resources to harvest", + ) + + @mock.patch("geonode.harvesting.tasks.finish_asynchronous_session") + @mock.patch("geonode.harvesting.tasks.logger") + @mock.patch("geonode.harvesting.tasks.models") + def test_harvest_resources_harvester_unavailable(self, mock_models, mock_logger, mock_finish): + mock_session = mock.MagicMock() + mock_session.status = mock_session.STATUS_ON_GOING + mock_session.harvester.name = "TestHarvester" + mock_session.harvester.remote_url = "http://remote" + mock_session.harvester.update_availability.return_value = False + mock_models.AsynchronousHarvestingSession.objects.get.return_value = mock_session + + tasks.harvest_resources([1, 2, 3], self.harvesting_session.id) + + mock_logger.warning.assert_called_once_with( + "Skipping harvesting for harvester 'TestHarvester' because remote 'http://remote' is unavailable" + ) + mock_finish.assert_called_once_with( + self.harvesting_session.id, + mock_session.STATUS_FINISHED_ALL_FAILED, + final_details=mock_logger.warning.call_args[0][0], + ) + + @mock.patch("geonode.harvesting.tasks.chord") + def test_queue_next_chunk_batch_schedules_next_batch(self, mock_chord): + chunk_groups = [[[1, 2], [3]], [[4, 5]]] + session_id = 42 + batch_index = 0 + dynamic_expiration = 900 + dynamic_time_limit = 1800 + + mock_chord_obj = mock.Mock() + mock_chord.return_value = mock_chord_obj + + tasks.queue_next_chunk_batch( + chunk_groups=chunk_groups, + harvesting_session_id=session_id, + batch_index=batch_index, + dynamic_expiration=dynamic_expiration, + dynamic_time_limit=dynamic_time_limit, + ) + + # Check that chord() was called + self.assertEqual(mock_chord.call_count, 3) + chords_args, chords_kwargs = mock_chord.call_args + + chords_in_batch = chords_args[0] + next_batch = chords_kwargs["body"] + + # Expect 2 chunks in batch 0 → 2 chords + self.assertEqual(len(chords_in_batch), 2) + + # Check next_batch is queue_next_chunk_batch task for batch 1 + self.assertEqual(next_batch.args[2], 1) + + # Final: check apply_async was called + mock_chord.return_value.apply_async.assert_called_once() + + @mock.patch("geonode.harvesting.tasks.chord") + @mock.patch("geonode.harvesting.tasks._finish_harvesting") + @mock.patch("geonode.harvesting.tasks._handle_harvesting_error") + def test_queue_next_chunk_batch_adds_global_finalizer_on_last_batch( + self, mock_handle_error, mock_finish_harvesting, mock_chord + ): + chunk_groups = [[[1, 2], [3]], [[4, 5]]] + session_id = 42 + batch_index = 1 + dynamic_expiration = 900 + dynamic_time_limit = 1800 + + mock_chord_obj = mock.Mock() + mock_chord.return_value = mock_chord_obj + + tasks.queue_next_chunk_batch( + chunk_groups=chunk_groups, + harvesting_session_id=session_id, + batch_index=batch_index, + dynamic_expiration=dynamic_expiration, + dynamic_time_limit=dynamic_time_limit, + ) + + # Ensure global finalizer was built and passed to chords_in_batch + self.assertTrue(mock_finish_harvesting.s.called) + self.assertTrue(mock_handle_error.s.called) + + chord_args, chord_kwargs = mock_chord.call_args + chords_in_batch = chord_args[0] + + # Expect 1 chunk + 1 global finalizer = 2 entries in chords_in_batch + self.assertEqual(len(chords_in_batch), 2) + self.assertIn(mock_finish_harvesting.s.return_value.on_error.return_value.set.return_value, chords_in_batch) + + # Final: check apply_async was called + mock_chord.return_value.apply_async.assert_called_once() diff --git a/geonode/harvesting/tests/test_utils.py b/geonode/harvesting/tests/test_utils.py new file mode 100644 index 0000000..cd520ad --- /dev/null +++ b/geonode/harvesting/tests/test_utils.py @@ -0,0 +1,50 @@ +############################################## +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from lxml import etree + +from geonode.tests.base import GeoNodeBaseSimpleTestSupport +from geonode.utils import get_xpath_value + + +class UtilsTestCase(GeoNodeBaseSimpleTestSupport): + def test_get_xpath_value(self): + fixtures = [ + ( + "phony", + "/ns1:myElement/ns2:anotherElement", + None, + "phony", + ), + ( + "phony", + "ns2:anotherElement", + None, + "phony", + ), + ( + "phony", + "ns2:anotherElement/ns3:additional", + None, + "phony", + ), + ] + for element, xpath_expr, nsmap, expected in fixtures: + xml_el = etree.fromstring(element) + result = get_xpath_value(xml_el, xpath_expr, nsmap=nsmap) + self.assertEqual(result, expected) diff --git a/geonode/harvesting/urls.py b/geonode/harvesting/urls.py new file mode 100644 index 0000000..a59a29f --- /dev/null +++ b/geonode/harvesting/urls.py @@ -0,0 +1,20 @@ +######################################################################### +# +# Copyright (C) 2021 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +urlpatterns = [] diff --git a/geonode/invitations/__init__.py b/geonode/invitations/__init__.py new file mode 100644 index 0000000..68c7b7d --- /dev/null +++ b/geonode/invitations/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/invitations/forms.py b/geonode/invitations/forms.py new file mode 100644 index 0000000..2bdda2a --- /dev/null +++ b/geonode/invitations/forms.py @@ -0,0 +1,74 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django import forms +from django.utils.translation import gettext_lazy as _ +from django.contrib.auth import get_user_model + +from invitations.adapters import get_invitations_adapter +from invitations.exceptions import AlreadyInvited, AlreadyAccepted, UserRegisteredEmail +from invitations.utils import get_invitation_model + +Invitation = get_invitation_model() + + +class CleanEmailMixin: + def validate_invitation(self, email): + if Invitation.objects.all_valid().filter(email__iexact=email, accepted=False): + raise AlreadyInvited + elif Invitation.objects.filter(email__iexact=email, accepted=True): + raise AlreadyAccepted + elif get_user_model().objects.filter(email__iexact=email): + raise UserRegisteredEmail + else: + return True + + def clean_email(self): + emails = self.cleaned_data["email"] + emails = emails.replace(";", ",").split(",") + for em in emails: + email = get_invitations_adapter().clean_email(em.strip()) + + errors = { + "already_invited": _(f"The e-mail address '{email}' has already been invited."), + "already_accepted": _(f"The e-mail address '{email}' has already accepted an invite."), + "email_in_use": _(f"An active user is already using the e-mail address '{email}'"), + } + try: + self.validate_invitation(email) + except AlreadyInvited: + raise forms.ValidationError(errors["already_invited"]) + except AlreadyAccepted: + raise forms.ValidationError(errors["already_accepted"]) + except UserRegisteredEmail: + raise forms.ValidationError(errors["email_in_use"]) + + return emails + + +class GeoNodeInviteForm(forms.Form, CleanEmailMixin): + email = forms.CharField( + label=_("E-mail"), required=True, widget=forms.TextInput(attrs={"type": "text", "size": "1200"}), initial="" + ) + + def save(self, emails): + invitations = [] + for em in emails: + invitations.append(Invitation.create(email=em.strip())) + return invitations diff --git a/geonode/invitations/migrations/__init__.py b/geonode/invitations/migrations/__init__.py new file mode 100644 index 0000000..68c7b7d --- /dev/null +++ b/geonode/invitations/migrations/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/invitations/urls.py b/geonode/invitations/urls.py new file mode 100644 index 0000000..5995eee --- /dev/null +++ b/geonode/invitations/urls.py @@ -0,0 +1,30 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.urls import re_path + +import invitations +from . import views + +app_name = "geonode.invitations" +urlpatterns = [ + re_path(r"^geonode-send-invite/$", views.GeoNodeSendInvite.as_view(), name="geonode-send-invite"), + re_path(r"^send-json-invite/$", invitations.views.SendJSONInvite.as_view(), name="send-json-invite"), + re_path(r"^accept-invite/(?P\w+)/?$", invitations.views.AcceptInvite.as_view(), name="accept-invite"), +] diff --git a/geonode/invitations/views.py b/geonode/invitations/views.py new file mode 100644 index 0000000..9fe43af --- /dev/null +++ b/geonode/invitations/views.py @@ -0,0 +1,112 @@ +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import traceback + +from django.contrib.sites.models import Site + +try: + from django.urls import reverse +except ImportError: + from django.urls import reverse +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from django.contrib.auth.decorators import login_required + +from .forms import GeoNodeInviteForm +from invitations import signals +from invitations.views import SendInvite +from invitations.utils import get_invitation_model +from invitations.adapters import get_invitations_adapter +from geonode.decorators import view_decorator + +Invitation = get_invitation_model() + + +@view_decorator(login_required, subclass=True) +class GeoNodeSendInvite(SendInvite): + template_name = "invitations/forms/_invite.html" + form_class = GeoNodeInviteForm + + def __init__(self, *args, **kwargs): + super(SendInvite, self).__init__(*args, **kwargs) + + def form_valid(self, form): + emails = form.cleaned_data["email"] + invited = [] + + invite = None + try: + invites = form.save(emails) + for invite_obj in invites: + invite = invite_obj + invite.inviter = self.request.user + invite.save() + # invite.send_invitation(self.request) + self.send_invitation(invite, self.request) + invited.append(invite_obj.email) + except Exception as e: + traceback.print_exc() + if invite: + invite.delete() + return self.form_invalid(form, emails, e) + + return self.render_to_response( + self.get_context_data( + success_message=_("Invitations successfully sent to '%(email)s'") % {"email": ", ".join(invited)} + ) + ) + + def form_invalid(self, form, emails=None, e=None): + if e: + return self.render_to_response( + self.get_context_data( + error_message=_( + "Sorry, it was not possible to invite '%(email)s'" + " due to the following isse: %(error)s (%(type)s)" + ) + % {"email": emails, "error": str(e), "type": type(e)} + ) + ) + else: + return self.render_to_response(self.get_context_data(form=form)) + + def send_invitation(self, invite, request, **kwargs): + current_site = kwargs.pop("site", Site.objects.get_current()) + invite_url = reverse("geonode.invitations:accept-invite", args=[invite.key]) + invite_url = request.build_absolute_uri(invite_url) + ctx = kwargs + ctx.update( + { + "invite_url": invite_url, + "site_name": current_site.name, + "email": invite.email, + "key": invite.key, + "inviter": invite.inviter, + } + ) + + email_template = "invitations/email/email_invite" + adapter = get_invitations_adapter() + adapter.send_invitation_email(email_template, invite.email, ctx) + invite.sent = timezone.now() + invite.save() + + signals.invite_url_sent.send( + sender=invite.__class__, instance=invite, invite_url_sent=invite_url, inviter=invite.inviter + ) diff --git a/geonode/layers/__init__.py b/geonode/layers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/layers/admin.py b/geonode/layers/admin.py new file mode 100644 index 0000000..7cb7122 --- /dev/null +++ b/geonode/layers/admin.py @@ -0,0 +1,130 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib import admin +from django.db.models import Prefetch + +from modeltranslation.admin import TabbedTranslationAdmin + +from geonode.base.admin import ResourceBaseAdminForm +from geonode.layers.models import Dataset, Attribute, Style +from geonode.base.admin import metadata_batch_edit + +from geonode.base.fields import MultiThesauriField +from geonode.base.models import ThesaurusKeyword, ThesaurusKeywordLabel + +from dal import autocomplete + + +class AttributeInline(admin.TabularInline): + model = Attribute + + +class DatasetAdminForm(ResourceBaseAdminForm): + class Meta(ResourceBaseAdminForm.Meta): + model = Dataset + fields = "__all__" + + tkeywords = MultiThesauriField( + queryset=ThesaurusKeyword.objects.prefetch_related( + Prefetch("keyword", queryset=ThesaurusKeywordLabel.objects.filter(lang="en")) + ), + widget=autocomplete.ModelSelect2Multiple( + url="thesaurus_autocomplete", + ), + label=("Keywords from Thesaurus"), + required=False, + help_text=("List of keywords from Thesaurus",), + ) + + +class DatasetAdmin(TabbedTranslationAdmin): + exclude = ("ll_bbox_polygon", "bbox_polygon", "srid") + list_display = ( + "id", + "alternate", + "title", + "date", + "category", + "group", + "is_approved", + "is_published", + "state", + "dirty_state", + "metadata_completeness", + ) + list_display_links = ("id",) + list_editable = ("title", "category", "group", "is_approved", "is_published", "dirty_state") + list_filter = ( + "subtype", + "owner", + "category", + "group", + "restriction_code_type__identifier", + "date", + "date_type", + "is_approved", + "is_published", + "state", + "dirty_state", + ) + search_fields = ("alternate", "title", "abstract", "purpose", "is_approved", "is_published", "state") + filter_horizontal = ("contacts",) + date_hierarchy = "date" + readonly_fields = ("uuid", "alternate", "workspace", "geographic_bounding_box") + inlines = [AttributeInline] + form = DatasetAdminForm + actions = [metadata_batch_edit] + + def delete_queryset(self, request, queryset): + """ + We need to invoke the 'ResourceBase.delete' method even when deleting + through the admin batch action + """ + for obj in queryset: + from geonode.resource.manager import resource_manager + + resource_manager.delete(obj.uuid, instance=obj) + + +class AttributeAdmin(admin.ModelAdmin): + model = Attribute + list_display_links = ("id",) + list_display = ("id", "dataset", "attribute", "description", "attribute_label", "attribute_type", "display_order") + list_filter = ("dataset", "attribute_type") + search_fields = ( + "attribute", + "attribute_label", + ) + + +class StyleAdmin(admin.ModelAdmin): + model = Style + list_display_links = ("sld_title",) + list_display = ("id", "name", "sld_title", "workspace", "sld_url") + list_filter = ("workspace",) + search_fields = ( + "name", + "workspace", + ) + + +admin.site.register(Dataset, DatasetAdmin) +admin.site.register(Attribute, AttributeAdmin) +admin.site.register(Style, StyleAdmin) diff --git a/geonode/layers/api/__init__.py b/geonode/layers/api/__init__.py new file mode 100644 index 0000000..0044807 --- /dev/null +++ b/geonode/layers/api/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/layers/api/exceptions.py b/geonode/layers/api/exceptions.py new file mode 100644 index 0000000..01d3d86 --- /dev/null +++ b/geonode/layers/api/exceptions.py @@ -0,0 +1,47 @@ +######################################################################### +# +# Copyright (C) 2022 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from rest_framework.exceptions import APIException + + +class GeneralDatasetException(APIException): + status_code = 500 + default_detail = "Error during dataset replace." + default_code = "dataset_exception" + category = "dataset_api" + + +class InvalidDatasetException(APIException): + status_code = 500 + default_detail = "Input payload is not valid" + default_code = "invalid_dataset_exception" + category = "dataset_api" + + +class InvalidMetadataException(APIException): + status_code = 500 + default_detail = "Input payload is not valid" + default_code = "invalid_metadata_exception" + category = "dataset_api" + + +class MissingMetadataException(APIException): + status_code = 400 + default_detail = "Metadata is missing" + default_code = "missing_metadata_exception" + category = "dataset_api" diff --git a/geonode/layers/api/permissions.py b/geonode/layers/api/permissions.py new file mode 100644 index 0000000..8ace585 --- /dev/null +++ b/geonode/layers/api/permissions.py @@ -0,0 +1,65 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from distutils.util import strtobool +from django.conf import settings +from rest_framework.filters import BaseFilterBackend + + +class DatasetPermissionsFilter(BaseFilterBackend): + """ + A filter backend that limits results to those where the requesting user + has read object level permissions. + """ + + shortcut_kwargs = { + "accept_global_perms": True, + } + + def filter_queryset(self, request, queryset, view): + # We want to defer this import until runtime, rather than import-time. + # See https://github.com/encode/django-rest-framework/issues/4608 + # (Also see #1624 for why we need to make this import explicitly) + from guardian.shortcuts import get_objects_for_user + from geonode.security.utils import get_visible_resources + + user = request.user + # perm_format = '%(app_label)s.view_%(model_name)s' + # permission = self.perm_format % { + # 'app_label': queryset.model._meta.app_label, + # 'model_name': queryset.model._meta.model_name, + # } + + resources = get_objects_for_user(user, "base.view_resourcebase", **self.shortcut_kwargs).filter( + polymorphic_ctype__model="dataset" + ) + try: + include_dirty = strtobool(request.query_params.get("include_dirty", "False")) + except Exception: + include_dirty = False + + obj_with_perms = get_visible_resources( + resources, + user, + admin_approval_required=settings.ADMIN_MODERATE_UPLOADS, + unpublished_not_visible=settings.RESOURCE_PUBLISHING, + private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES, + include_dirty=include_dirty, + ) + + return queryset.filter(id__in=obj_with_perms.values("id")) diff --git a/geonode/layers/api/serializers.py b/geonode/layers/api/serializers.py new file mode 100644 index 0000000..b061f3a --- /dev/null +++ b/geonode/layers/api/serializers.py @@ -0,0 +1,215 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from rest_framework import serializers + +from urllib.parse import urlparse + +from django.conf import settings + +from dynamic_rest.serializers import DynamicModelSerializer +from dynamic_rest.fields.fields import DynamicRelationField, DynamicComputedField + +from geonode.layers.models import Dataset, Style, Attribute +from geonode.base.api.serializers import ResourceBaseSerializer + +import logging + +logger = logging.getLogger(__name__) + + +class StyleSerializer(DynamicModelSerializer): + class Meta: + model = Style + name = "style" + fields = ("pk", "name", "workspace", "sld_title", "sld_url") + + name = serializers.CharField(read_only=True) + workspace = serializers.CharField(read_only=True) + sld_url = serializers.SerializerMethodField() + + def get_sld_url(self, instance): + if bool(urlparse(instance.sld_url).netloc): + return instance.sld_url.replace( + settings.OGC_SERVER["default"]["LOCATION"], settings.OGC_SERVER["default"]["PUBLIC_LOCATION"] + ) + return instance.sld_url + + +class AttributeSerializer(DynamicModelSerializer): + class Meta: + model = Attribute + name = "attribute" + fields = ( + "pk", + "attribute", + "description", + "attribute_label", + "attribute_type", + "visible", + "display_order", + "featureinfo_type", + "count", + "min", + "max", + "average", + "median", + "stddev", + "sum", + "unique_values", + "last_stats_updated", + ) + + attribute = serializers.CharField(read_only=True) + + +class FeatureInfoTemplateField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + if instance.use_featureinfo_custom_template and instance.featureinfo_custom_template: + return instance.featureinfo_custom_template + else: + _attributes = instance.attributes.filter(visible=True).order_by("display_order") + if _attributes.exists(): + _template = '
      ' + for _field in _attributes: + _label = _field.attribute_label or _field.attribute + _template += '
      ' + if _field.featureinfo_type == Attribute.TYPE_HREF: + _template += ( + '
      %s:
      \ +
      ${properties[\'%s\']}
      ' + % (_label, _field.attribute, _field.attribute) + ) + elif _field.featureinfo_type == Attribute.TYPE_IMAGE: + _template += ( + '
      \ + %s
      ' + % (_field.attribute, _field.attribute, _label, _label) + ) + elif _field.featureinfo_type in ( + Attribute.TYPE_VIDEO_3GP, + Attribute.TYPE_VIDEO_FLV, + Attribute.TYPE_VIDEO_MP4, + Attribute.TYPE_VIDEO_OGG, + Attribute.TYPE_VIDEO_WEBM, + Attribute.TYPE_VIDEO_YOUTUBE, + ): + if "youtube" in _field.featureinfo_type: + _template += ( + '
      \ +
      ' + % (_field.attribute) + ) + else: + _type = f"video/{_field.featureinfo_type[11:]}" + _template += ( + '
      \ +
      ' + % (_field.attribute, _type) + ) + elif _field.featureinfo_type == Attribute.TYPE_AUDIO: + _template += ( + '
      \ +
      ' + % (_field.attribute) + ) + elif _field.featureinfo_type == Attribute.TYPE_IFRAME: + _template += ( + '
      \ +
      ' + % (_field.attribute) + ) + elif _field.featureinfo_type == Attribute.TYPE_PROPERTY: + _template += ( + '
      %s:
      \ +
      ${properties[\'%s\']}
      ' + % (_label, _field.attribute) + ) + _template += "
      " + _template += "
      " + return _template + return None + + +class DatasetSerializer(ResourceBaseSerializer): + name = serializers.CharField(read_only=True) + workspace = serializers.CharField(read_only=True) + store = serializers.CharField(read_only=True) + charset = serializers.CharField(read_only=True) + + default_style = DynamicRelationField(StyleSerializer, embed=True, many=False, read_only=True) + styles = DynamicRelationField(StyleSerializer, embed=True, many=True, read_only=True) + + attribute_set = DynamicRelationField(AttributeSerializer, embed=True, many=True, read_only=True) + featureinfo_custom_template = FeatureInfoTemplateField() + + class Meta: + model = Dataset + name = "dataset" + view_name = "datasets-list" + fields = list( + set( + ResourceBaseSerializer.Meta.fields + + ( + "uuid", + "name", + "metadata", + "attribute_set", + "charset", + "is_mosaic", + "has_time", + "has_elevation", + "time_regex", + "elevation_regex", + "featureinfo_custom_template", + "ows_url", + "capabilities_url", + "dataset_ows_url", + "workspace", + "default_style", + "styles", + "store", + "subtype", + "ptype", + ) + ) + ) + + +class DatasetListSerializer(DatasetSerializer): + class Meta(DatasetSerializer.Meta): + fields = [ + f + for f in DatasetSerializer.Meta.fields + if f not in ("attribute_set", "capabilities_url", "dataset_ows_url", "ows_url") + ] + + +class MetadataFileField(DynamicComputedField): + def get_attribute(self, instance): + return instance.get("metadata_file") + + +class DatasetMetadataSerializer(serializers.Serializer): + metadata_file = MetadataFileField(required=True) + + class Meta: + fields = "metadata_file" diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py new file mode 100644 index 0000000..a955fb6 --- /dev/null +++ b/geonode/layers/api/tests.py @@ -0,0 +1,697 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from io import BytesIO +import logging + +from django.contrib.auth import get_user_model +from urllib.parse import urljoin + +from django.conf import settings +from django.test import override_settings +from django.urls import reverse +from rest_framework.test import APITestCase + +from guardian.shortcuts import assign_perm, get_anonymous_user + +from geonode.base.models import Link +from geonode.base.populate_test_data import create_models, create_single_dataset +from geonode.layers.models import Attribute, Dataset +from geonode.maps.models import Map, MapLayer + +logger = logging.getLogger(__name__) + + +class DatasetsApiTests(APITestCase): + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + def setUp(self): + self.exml_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" + create_models(b"document") + create_models(b"map") + create_models(b"dataset") + + def test_datasets(self): + """ + Ensure we can access the Layers list. + """ + url = reverse("datasets-list") + # Anonymous + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 8) + + # Pagination + self.assertEqual(len(response.data["datasets"]), 8) + logger.debug(response.data) + + for _l in response.data["datasets"]: + self.assertTrue(_l["resource_type"], "dataset") + # Test list response doesn't have attribute_set + self.assertIsNotNone(response.data["datasets"][0].get("ptype")) + self.assertIsNone(response.data["datasets"][0].get("attribute_set")) + self.assertIsNone(response.data["datasets"][0].get("featureinfo_custom_template")) + + _dataset = Dataset.objects.first() + assign_perm("base.view_resourcebase", get_anonymous_user(), _dataset.get_self_resource()) + + # Test detail response has attribute_set + url = urljoin(f"{reverse('datasets-list')}/", f"{_dataset.pk}") + response = self.client.get(url, format="json") + self.assertIsNotNone(response.data["dataset"].get("ptype")) + self.assertIsNotNone(response.data["dataset"].get("subtype")) + self.assertIsNotNone(response.data["dataset"].get("attribute_set")) + + # Test "featureinfo_custom_template" + _attribute, _ = Attribute.objects.get_or_create(dataset=_dataset, attribute="name") + try: + _attribute.visible = True + _attribute.attribute_type = Attribute.TYPE_PROPERTY + _attribute.description = "The Name" + _attribute.attribute_label = "Name" + _attribute.display_order = 1 + _attribute.save() + + url = urljoin(f"{reverse('datasets-list')}/", f"{_dataset.pk}") + response = self.client.get(url, format="json") + self.assertIsNotNone(response.data["dataset"].get("featureinfo_custom_template")) + self.assertEqual( + response.data["dataset"].get("featureinfo_custom_template"), + '
      Name:
      \ +
      ${properties[\'name\']}
      ', + ) + + _dataset.featureinfo_custom_template = "
      Foo Bar
      " + _dataset.save() + url = urljoin(f"{reverse('datasets-list')}/", f"{_dataset.pk}") + response = self.client.get(url, format="json") + self.assertIsNotNone(response.data["dataset"].get("featureinfo_custom_template")) + self.assertEqual( + response.data["dataset"].get("featureinfo_custom_template"), + '
      Name:
      \ +
      ${properties[\'name\']}
      ', + ) + + _dataset.use_featureinfo_custom_template = True + _dataset.save() + url = urljoin(f"{reverse('datasets-list')}/", f"{_dataset.pk}") + response = self.client.get(url, format="json") + self.assertIsNotNone(response.data["dataset"].get("featureinfo_custom_template")) + self.assertEqual(response.data["dataset"].get("featureinfo_custom_template"), "
      Foo Bar
      ") + finally: + _attribute.delete() + _dataset.featureinfo_custom_template = None + _dataset.use_featureinfo_custom_template = False + _dataset.save() + + @override_settings(REST_API_DEFAULT_PAGE_SIZE=100) + def test_filter_dirty_state(self): + """ + ensure that a dirty_state dataset wont be returned + """ + + # ensure that there is atleast one resource with dirty_state + dirty_dataset = Dataset.objects.first() + dirty_dataset.dirty_state = True + dirty_dataset.save() + + url = reverse("datasets-list") + + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + dataset_list = response.data["datasets"] + + # ensure that list count is equal to that of clean data + # clean resources + resource_count_clean = Dataset.objects.filter(dirty_state=False).count() + self.assertEqual(len(dataset_list), resource_count_clean) + # ensure that the updated dirty dataset is not in the response + self.assertFalse(dirty_dataset.pk in [int(dataset["pk"]) for dataset in dataset_list]) + + @override_settings(REST_API_DEFAULT_PAGE_SIZE=100) + def test_filter_dirty_state_include_dirty(self): + """ + ensure that all resources are returned when dirty_state is true + """ + # ensure that there is atleast one resource with dirty_state + dirty_dataset = Dataset.objects.first() + dirty_dataset.dirty_state = True + dirty_dataset.save() + + # clean resources + resource_count_clean = Dataset.objects.filter(dirty_state=False).count() + # dirty resources + resource_count_dirty = Dataset.objects.filter(dirty_state=True).count() + + resource_count_all = resource_count_clean + resource_count_dirty + + url = f'{reverse("datasets-list")}?include_dirty=true' + + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + dataset_list = response.data["datasets"] + + # ensure that list count is equal to that of all data + self.assertEqual(len(dataset_list), resource_count_all) + + # ensure that the updated dirty dataset is in the response + self.assertTrue(dirty_dataset.pk in [int(dataset["pk"]) for dataset in dataset_list]) + + def test_dataset_listing_advertised(self): + app = Dataset.objects.first() + app.advertised = False + app.save() + + url = reverse("datasets-list") + + payload = self.client.get(url) + + prev_count = payload.json().get("total") + # the user can see only the advertised resources + self.assertEqual(Dataset.objects.filter(advertised=True).count(), prev_count) + + payload = self.client.get(f"{url}?advertised=True") + # so if advertised is True, we dont see the advertised=False resource + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count) + + payload = self.client.get(f"{url}?advertised=False") + # so if advertised is False, we see only the resource with advertised==False + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, 1) + + # if all is requested, we will see all the resources + payload = self.client.get(f"{url}?advertised=all") + new_count = payload.json().get("total") + # recheck the count + self.assertEqual(new_count, prev_count + 1) + + Dataset.objects.update(advertised=True) + + def test_extra_metadata_included_with_param(self): + _dataset = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{_dataset.pk}") + data = {"include[]": "metadata"} + + response = self.client.get(url, format="json", data=data) + self.assertIsNotNone(response.data["dataset"].get("metadata")) + + response = self.client.get(url, format="json") + self.assertNotIn("metadata", response.data["dataset"]) + + def test_get_dataset_related_maps_and_maplayers(self): + dataset = Dataset.objects.first() + assign_perm("base.view_resourcebase", get_anonymous_user(), dataset.get_self_resource()) + + url = reverse("datasets-detail", kwargs={"pk": dataset.pk}) + response = self.client.get(f"{url}/maplayers", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 0) + + response = self.client.get(f"{url}/maplayers", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 0) + + response = self.client.get(f"{url}/maps", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 0) + map = Map.objects.first() + map_layer = MapLayer.objects.create( + map=map, + extra_params={}, + name=dataset.alternate, + store=None, + current_style=None, + ows_url=None, + local=True, + dataset=dataset, + ) + response = self.client.get(f"{url}/maplayers", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 1) + self.assertEqual(response.json()[0]["pk"], map_layer.pk) + + response = self.client.get(f"{url}/maps", format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 1) + self.assertEqual(response.json()[0]["pk"], map.pk) + + def test_raw_HTML_stripped_properties(self): + """ + Ensure "raw_*" properties returns no HTML or carriage-return tag + """ + dataset = Dataset.objects.first() + dataset.abstract = ( + '

      No abstract provided.

      \r\n

      None

      ' + dataset.supplemental_information = "

      No information provided í

      £682m

      " + dataset.data_quality_statement = '

      OK

      \r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\ + \r\n\r\n\r\n
      12
      ab
      ' + dataset.save() + + # Admin + self.assertTrue(self.client.login(username="admin", password="admin")) + + url = reverse("datasets-detail", kwargs={"pk": dataset.pk}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(int(response.data["dataset"]["pk"]), int(dataset.pk)) + self.assertEqual(response.data["dataset"]["raw_abstract"], "No abstract provided.") + self.assertEqual(response.data["dataset"]["raw_constraints_other"], "None") + self.assertEqual(response.data["dataset"]["raw_supplemental_information"], "No information provided í £682m") + self.assertEqual(response.data["dataset"]["raw_data_quality_statement"], "OK 1 2 a b") + + def test_patch_point_of_contact(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"poc": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all(user_id in [poc.get("pk") for poc in response.json().get("dataset").get("poc")] for user_id in user_ids) + ) + # Resetting all point of contact + response = self.client.patch(url, data={"poc": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [poc.get("pk") for poc in response.json().get("dataset").get("poc")] + for user_id in user_ids + ) + ) + + def test_patch_metadata_author(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"metadata_author": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [ + metadata_author.get("pk") + for metadata_author in response.json().get("dataset").get("metadata_author") + ] + for user_id in user_ids + ) + ) + # Resetting all metadata authors + response = self.client.patch(url, data={"metadata_author": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + metadata_author.get("pk") + for metadata_author in response.json().get("dataset").get("metadata_author") + ] + for user_id in user_ids + ) + ) + + def test_patch_processor(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"processor": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [processor.get("pk") for processor in response.json().get("dataset").get("processor")] + for user_id in user_ids + ) + ) + # Resetting all processors + response = self.client.patch(url, data={"processor": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [processor.get("pk") for processor in response.json().get("dataset").get("processor")] + for user_id in user_ids + ) + ) + + def test_patch_publisher(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"publisher": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [publisher.get("pk") for publisher in response.json().get("dataset").get("publisher")] + for user_id in user_ids + ) + ) + # Resetting all publishers + response = self.client.patch(url, data={"publisher": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [publisher.get("pk") for publisher in response.json().get("dataset").get("publisher")] + for user_id in user_ids + ) + ) + + def test_patch_custodian(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"custodian": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [custodian.get("pk") for custodian in response.json().get("dataset").get("custodian")] + for user_id in user_ids + ) + ) + # Resetting all custodians + response = self.client.patch(url, data={"custodian": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [custodian.get("pk") for custodian in response.json().get("dataset").get("custodian")] + for user_id in user_ids + ) + ) + + def test_patch_distributor(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"distributor": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [distributor.get("pk") for distributor in response.json().get("dataset").get("distributor")] + for user_id in user_ids + ) + ) + # Resetting all distributers + response = self.client.patch(url, data={"distributor": []}, format="json") + + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [distributor.get("pk") for distributor in response.json().get("dataset").get("distributor")] + for user_id in user_ids + ) + ) + + def test_patch_resource_user(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"resource_user": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [resource_user.get("pk") for resource_user in response.json().get("dataset").get("resource_user")] + for user_id in user_ids + ) + ) + # Resetting all resource users + response = self.client.patch(url, data={"resource_user": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + resource_user.get("pk") for resource_user in response.json().get("dataset").get("resource_user") + ] + for user_id in user_ids + ) + ) + + def test_patch_resource_provider(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"resource_provider": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [ + resource_provider.get("pk") + for resource_provider in response.json().get("dataset").get("resource_provider") + ] + for user_id in user_ids + ) + ) + # Resetting all principal investigator + response = self.client.patch(url, data={"resource_provider": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + resource_provider.get("pk") + for resource_provider in response.json().get("dataset").get("resource_provider") + ] + for user_id in user_ids + ) + ) + + def test_patch_originator(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"originator": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id in [originator.get("pk") for originator in response.json().get("dataset").get("originator")] + for user_id in user_ids + ) + ) + # Resetting all originators + response = self.client.patch(url, data={"originator": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id not in [originator.get("pk") for originator in response.json().get("dataset").get("originator")] + for user_id in user_ids + ) + ) + + def test_patch_principal_investigator(self): + layer = Dataset.objects.first() + url = urljoin(f"{reverse('datasets-list')}/", f"{layer.id}") + self.client.login(username="admin", password="admin") + get_user_model().objects.get_or_create(username="ninja") + get_user_model().objects.get_or_create(username="turtle") + users = get_user_model().objects.exclude(pk=-1) + user_ids = [user.pk for user in users] + patch_data = {"principal_investigator": [{"pk": uid} for uid in user_ids]} + response = self.client.patch(url, data=patch_data, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + in [ + principal_investigator.get("pk") + for principal_investigator in response.json().get("dataset").get("principal_investigator") + ] + for user_id in user_ids + ) + ) + # Resetting all principal investigator + response = self.client.patch(url, data={"principal_investigator": []}, format="json") + self.assertEqual(200, response.status_code) + self.assertTrue( + all( + user_id + not in [ + principal_investigator.get("pk") + for principal_investigator in response.json().get("dataset").get("principal_investigator") + ] + for user_id in user_ids + ) + ) + + def test_metadata_update_for_not_supported_method(self): + layer = Dataset.objects.first() + url = reverse("datasets-replace-metadata", args=(layer.id,)) + self.client.login(username="admin", password="admin") + + response = self.client.post(url) + self.assertEqual(405, response.status_code) + + response = self.client.get(url) + self.assertEqual(405, response.status_code) + + def test_metadata_update_for_not_authorized_user(self): + layer = Dataset.objects.first() + url = reverse("datasets-replace-metadata", args=(layer.id,)) + + response = self.client.put(url) + self.assertEqual(403, response.status_code) + + def test_unsupported_file_throws_error(self): + layer = Dataset.objects.first() + url = reverse("datasets-replace-metadata", args=(layer.id,)) + self.client.login(username="admin", password="admin") + + data = '' + f = BytesIO(bytes(data, encoding="utf-8")) + f.name = "metadata.xml" + put_data = {"metadata_file": f} + response = self.client.put(url, data=put_data) + self.assertEqual(500, response.status_code) + + def test_valid_metadata_file_with_different_uuid(self): + layer = Dataset.objects.first() + url = reverse("datasets-replace-metadata", args=(layer.id,)) + self.client.login(username="admin", password="admin") + + f = open(self.exml_path, "r") + put_data = {"metadata_file": f} + response = self.client.put(url, data=put_data) + self.assertEqual(500, response.status_code) + + def test_permissions_for_not_permitted_user(self): + get_user_model().objects.create_user( + username="some_user", + password="some_password", + email="some_user@geonode.org", + ) + layer = Dataset.objects.first() + url = reverse("datasets-replace-metadata", args=(layer.id,)) + self.client.login(username="some_user", password="some_password") + + uuid = layer.uuid + data = open(self.exml_path).read() + data = data.replace("7cfbc42c-efa7-431c-8daa-1399dff4cd19", uuid) + f = BytesIO(bytes(data, encoding="utf-8")) + f.name = "metadata.xml" + put_data = {"metadata_file": f} + response = self.client.put(url, data=put_data) + self.assertEqual(403, response.status_code) + + def test_permissions_for_permitted_user(self): + another_non_admin_user = get_user_model().objects.create_user( + username="some_other_user", + password="some_other_password", + email="some_other_user@geonode.org", + ) + layer = Dataset.objects.first() + assign_perm("base.change_resourcebase_metadata", another_non_admin_user, layer.get_self_resource()) + url = reverse("datasets-replace-metadata", args=(layer.id,)) + self.client.login(username="some_other_user", password="some_other_password") + + uuid = layer.uuid + data = open(self.exml_path).read() + data = data.replace("7cfbc42c-efa7-431c-8daa-1399dff4cd19", uuid) + f = BytesIO(bytes(data, encoding="utf-8")) + f.name = "metadata.xml" + put_data = {"metadata_file": f} + response = self.client.put(url, data=put_data) + self.assertEqual(200, response.status_code) + + def test_valid_metadata_file(self): + layer = Dataset.objects.first() + url = reverse("datasets-replace-metadata", args=(layer.id,)) + self.client.login(username="admin", password="admin") + + uuid = layer.uuid + data = open(self.exml_path).read() + data = data.replace("7cfbc42c-efa7-431c-8daa-1399dff4cd19", uuid) + f = BytesIO(bytes(data, encoding="utf-8")) + f.name = "metadata.xml" + put_data = {"metadata_file": f} + response = self.client.put(url, data=put_data) + self.assertEqual(200, response.status_code) + + def test_download_api(self): + dataset = create_single_dataset("test_dataset") + url = reverse("datasets-detail", kwargs={"pk": dataset.pk}) + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + data = response.json()["dataset"] + download_url_data = data["download_urls"][0] + download_url = reverse("dataset_download", args=[dataset.alternate]) + self.assertEqual(download_url_data["default"], True) + self.assertEqual(download_url_data["ajax_safe"], True) + self.assertEqual(download_url_data["url"], download_url) + + link = Link(link_type="original", url="https://myoriginal.org", resource=dataset) + link.save() + + response = self.client.get(url) + data = response.json()["dataset"] + download_url_data = data["download_urls"][0] + download_url = reverse("dataset_download", args=[dataset.alternate]) + self.assertEqual(download_url_data["default"], True) + self.assertEqual(download_url_data["ajax_safe"], False) + self.assertEqual(download_url_data["url"], "https://myoriginal.org") diff --git a/geonode/layers/api/urls.py b/geonode/layers/api/urls.py new file mode 100644 index 0000000..5305a6d --- /dev/null +++ b/geonode/layers/api/urls.py @@ -0,0 +1,25 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.api.urls import router + +from . import views + +router.register(r"datasets", views.DatasetViewSet, "datasets") + +urlpatterns = [] diff --git a/geonode/layers/api/views.py b/geonode/layers/api/views.py new file mode 100644 index 0000000..35fdbc2 --- /dev/null +++ b/geonode/layers/api/views.py @@ -0,0 +1,189 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from drf_spectacular.utils import extend_schema + +from dynamic_rest.viewsets import DynamicModelViewSet +from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter + +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticatedOrReadOnly, IsAuthenticated +from rest_framework.authentication import SessionAuthentication, BasicAuthentication + +from oauth2_provider.contrib.rest_framework import OAuth2Authentication +from rest_framework.response import Response + +from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter +from geonode.base.api.mixins import AdvertisedListMixin +from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.base.api.permissions import UserHasPerms +from geonode.base.api.views import ApiPresetsInitializer +from geonode.layers.api.exceptions import GeneralDatasetException, InvalidDatasetException, InvalidMetadataException +from geonode.layers.metadata import parse_metadata +from geonode.layers.models import Dataset +from geonode.maps.api.serializers import SimpleMapLayerSerializer, SimpleMapSerializer +from geonode.resource.utils import update_resource +from geonode.resource.manager import resource_manager +from rest_framework.exceptions import NotFound + +from geonode.storage.manager import StorageManager + +from .serializers import ( + DatasetSerializer, + DatasetListSerializer, + DatasetMetadataSerializer, +) +from .permissions import DatasetPermissionsFilter + +import logging + +logger = logging.getLogger(__name__) + + +class DatasetViewSet(ApiPresetsInitializer, DynamicModelViewSet, AdvertisedListMixin): + """ + API endpoint that allows layers to be viewed or edited. + """ + + http_method_names = ["get", "patch", "put"] + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [ + IsAuthenticatedOrReadOnly, + UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}), + ] + filter_backends = [ + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + ExtentFilter, + DatasetPermissionsFilter, + ] + queryset = Dataset.objects.all().order_by("-created") + serializer_class = DatasetSerializer + pagination_class = GeoNodeApiPagination + + def get_serializer_class(self): + if self.action == "list": + return DatasetListSerializer + return DatasetSerializer + + def partial_update(self, request, *args, **kwargs): + result = super().partial_update(request, *args, **kwargs) + + dataset = self.get_object() + resource_manager.update(dataset.uuid, instance=dataset, notify=True), + + return result + + @extend_schema( + request=DatasetMetadataSerializer, + methods=["put"], + responses={200}, + description="API endpoint to upload metadata file.", + ) + @action( + detail=False, + url_path="(?P\d+)/metadata", # noqa + url_name="replace-metadata", + methods=["put"], + serializer_class=DatasetMetadataSerializer, + permission_classes=[ + IsAuthenticated, + UserHasPerms(perms_dict={"default": {"PUT": ["base.change_resourcebase_metadata"]}}), + ], + ) + def metadata(self, request, pk=None, *args, **kwargs): + """ + Endpoint to upload ISO metadata + Usage Example: + + import requests + + dataset_id = 1 + url = f"http://localhost:8080/api/v2/datasets/{dataset_id}/metadata" + files=[ + ('metadata_file',('metadata.xml',open('/home/user/metadata.xml','rb'),'text/xml')) + ] + headers = { + 'Authorization': 'Basic dXNlcjpwYXNzd29yZA==' + } + response = requests.request("PUT", url, payload={}, files=files) + + cURL example: + curl --location --request PUT 'http://localhost:8000/api/v2/datasets/{dataset_id}/metadata' \ + --form 'metadata_file=@/home/user/metadata.xml' + """ + out = {} + storage_manager = None + if not self.queryset.filter(id=pk).exists(): + raise NotFound(detail=f"Dataset with ID {pk} is not available") + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(raise_exception=False): + raise InvalidDatasetException(detail=serializer.errors) + try: + data = serializer.data.copy() + if not data["metadata_file"]: + raise InvalidMetadataException(detail="A valid metadata file must be specified") + storage_manager = StorageManager(remote_files=data) + storage_manager.clone_remote_files() + file = storage_manager.get_retrieved_paths() + metadata_file = file["metadata_file"] + dataset = self.queryset.get(id=pk) + try: + dataset_uuid, vals, regions, keywords, _ = parse_metadata(open(metadata_file).read()) + except Exception: + raise InvalidMetadataException(detail="Unsupported metadata format") + if dataset_uuid and dataset.uuid != dataset_uuid: + raise InvalidMetadataException( + detail="The UUID identifier from the XML Metadata, is different from the one saved" + ) + try: + updated_dataset = update_resource(dataset, metadata_file, regions, keywords, vals) + updated_dataset.save() # This also triggers the recreation of the XML metadata file according to the updated values + except Exception: + raise GeneralDatasetException(detail="Failed to update metadata") + out["success"] = True + out["message"] = ["Metadata successfully updated"] + return Response(out) + except Exception as e: + raise e + finally: + if storage_manager: + storage_manager.delete_retrieved_paths() + + @extend_schema( + methods=["get"], + responses={200: SimpleMapLayerSerializer(many=True)}, + description="API endpoint allowing to retrieve the MapLayers list.", + ) + @action(detail=True, methods=["get"]) + def maplayers(self, request, pk=None, *args, **kwargs): + dataset = self.get_object() + resources = dataset.maplayers + return Response(SimpleMapLayerSerializer(many=True).to_representation(resources)) + + @extend_schema( + methods=["get"], + responses={200: SimpleMapSerializer(many=True)}, + description="API endpoint allowing to retrieve maps using the dataset.", + ) + @action(detail=True, methods=["get"]) + def maps(self, request, pk=None, *args, **kwargs): + dataset = self.get_object() + resources = dataset.maps + return Response(SimpleMapSerializer(many=True).to_representation(resources)) diff --git a/geonode/layers/apps.py b/geonode/layers/apps.py new file mode 100644 index 0000000..f7c5031 --- /dev/null +++ b/geonode/layers/apps.py @@ -0,0 +1,63 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.apps import AppConfig +from django.utils.translation import gettext_noop as _ +from geonode.notifications_helper import NotificationsAppConfigBase + + +class DatasetAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.layers" + verbose_name = "Dataset" + verbose_name_plural = "Datasets" + NOTIFICATIONS = ( + ( + "dataset_created", + _("Dataset Created"), + _("A Dataset was created"), + ), + ( + "dataset_updated", + _("Dataset Updated"), + _("A Dataset was updated"), + ), + ( + "dataset_approved", + _("Dataset Approved"), + _("A Dataset was approved by a Manager"), + ), + ( + "dataset_published", + _("Dataset Published"), + _("A Dataset was published"), + ), + ( + "dataset_deleted", + _("Dataset Deleted"), + _("A Dataset was deleted"), + ), + ( + "dataset_rated", + _("Rating for Dataset"), + _("A rating was given to a layer"), + ), + ) + + +default_app_config = "geonode.layers.DatasetAppConfig" diff --git a/geonode/layers/download_handler.py b/geonode/layers/download_handler.py new file mode 100644 index 0000000..7f154a6 --- /dev/null +++ b/geonode/layers/download_handler.py @@ -0,0 +1,168 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import xml.etree.ElementTree as ET + +from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse +from django.template.loader import get_template +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.conf import settings +from geonode.base.auth import get_or_create_token +from geonode.geoserver.helpers import wps_format_is_supported +from geonode.layers.views import _resolve_dataset +from geonode.proxy.views import fetch_response_headers +from geonode.utils import HttpClient + +logger = logging.getLogger("geonode.layers.download_handler") + + +class DatasetDownloadHandler: + def __str__(self): + return f"{self.__module__}.{self.__class__.__name__}" + + def __repr__(self): + return self.__str__() + + def __init__(self, request, resource_name) -> None: + self.request = request + self.resource_name = resource_name + self._resource = None + + def get_download_response(self): + """ + Basic method. Should return the Response object + that allow the resource download + """ + resource = self.get_resource() + if not resource: + raise Http404("Resource requested is not available") + response = self.process_dowload(resource) + return response + + @property + def is_link_resource(self): + resource = self.get_resource() + return resource.link_set.filter(resource=resource, link_type="original").exists() + + @property + def is_ajax_safe(self): + """ + AJAX is safe to be used for WPS downloads. In case of a link set in a Link entry we cannot assume it, + since it could point to an external (non CORS enabled) URL + """ + return settings.USE_GEOSERVER and not self.is_link_resource + + @property + def download_url(self): + resource = self.get_resource() + if not resource: + return None + if resource.subtype not in ["vector", "raster", "vector_time"]: + logger.info("Download URL is available only for datasets that have been harvested and copied locally") + return None + + if self.is_link_resource: + return resource.link_set.filter(resource=resource.get_self_resource(), link_type="original").first().url + + return reverse("dataset_download", args=[resource.alternate]) + + def get_resource(self): + """ + Returnt the object needed + """ + if not self._resource: + try: + self._resource = _resolve_dataset( + self.request, + self.resource_name, + "base.download_resourcebase", + _("You do not have download permissions for this dataset."), + ) + except Exception as e: + logger.debug(e) + + return self._resource + + def process_dowload(self, resource=None): + """ + Generate the response object + """ + if not resource: + resource = self.get_resource() + if not settings.USE_GEOSERVER: + # if GeoServer is not used, we redirect to the proxy download + return HttpResponseRedirect(reverse("download", args=[resource.id])) + + download_format = self.request.GET.get("export_format") + + if download_format and not wps_format_is_supported(download_format, resource.subtype): + logger.error("The format provided is not valid for the selected resource") + return JsonResponse({"error": "The format provided is not valid for the selected resource"}, status=500) + + _format = "application/zip" if resource.is_vector() else "image/tiff" + # getting default payload + tpl = get_template("geoserver/dataset_download.xml") + ctx = {"alternate": resource.alternate, "download_format": download_format or _format} + # applying context for the payload + payload = tpl.render(ctx) + + # init of Client + client = HttpClient() + + headers = {"Content-type": "application/xml", "Accept": "application/xml"} + + # defining the URL needed fr the download + url = f"{settings.OGC_SERVER['default']['LOCATION']}ows?service=WPS&version=1.0.0&REQUEST=Execute" + if not self.request.user.is_anonymous: + # define access token for the user + access_token = get_or_create_token(self.request.user) + url += f"&access_token={access_token}" + + # request to geoserver + response, content = client.request(url=url, data=payload, method="post", headers=headers) + + if not response or response.status_code != 200: + logger.error(f"Download dataset exception: error during call with GeoServer: {content}") + return JsonResponse( + {"error": "Download dataset exception: error during call with GeoServer"}, + status=500, + ) + + # error handling + namespaces = {"ows": "http://www.opengis.net/ows/1.1", "wps": "http://www.opengis.net/wps/1.0.0"} + response_type = response.headers.get("Content-Type") + if response_type == "text/xml": + # parsing XML for get exception + content = ET.fromstring(response.text) + exc = content.find("*//ows:Exception", namespaces=namespaces) or content.find( + "ows:Exception", namespaces=namespaces + ) + if exc: + exc_text = exc.find("ows:ExceptionText", namespaces=namespaces) + logger.error(f"{exc.attrib.get('exceptionCode')} {exc_text.text}") + return JsonResponse({"error": f"{exc.attrib.get('exceptionCode')}: {exc_text.text}"}, status=500) + + return_response = fetch_response_headers( + HttpResponse(content=response.content, status=response.status_code, content_type=download_format), + response.headers, + ) + return_response.headers["Content-Type"] = download_format or _format + return return_response diff --git a/geonode/layers/enumerations.py b/geonode/layers/enumerations.py new file mode 100644 index 0000000..e0b0df3 --- /dev/null +++ b/geonode/layers/enumerations.py @@ -0,0 +1,49 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +LAYER_ATTRIBUTE_NUMERIC_DATA_TYPES = [ + "xsd:byte", + "xsd:decimal", + "xsd:double", + "xsd:int", + "xsd:integer", + "xsd:long", + "xsd:negativeInteger", + "xsd:nonNegativeInteger", + "xsd:nonPositiveInteger", + "xsd:positiveInteger", + "xsd:short", + "xsd:unsignedLong", + "xsd:unsignedInt", + "xsd:unsignedShort", + "xsd:unsignedByte", +] + +GXP_PTYPES = { + "AUTO": "gxp_wmscsource", + "OWS": "gxp_wmscsource", + "WMS": "gxp_wmscsource", + "WFS": "gxp_wmscsource", + "WCS": "gxp_wmscsource", + "REST_MAP": "gxp_arcrestsource", + "REST_IMG": "gxp_arcrestsource", + "HGL": "gxp_hglsource", + "GN_WMS": "gxp_wmscsource", + "SOS": "gxp_sos", +} diff --git a/geonode/layers/forms.py b/geonode/layers/forms.py new file mode 100644 index 0000000..38db3c7 --- /dev/null +++ b/geonode/layers/forms.py @@ -0,0 +1,151 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django import forms + +import json +from geonode.base.forms import ResourceBaseForm, get_tree_data +from geonode.layers.models import Dataset, Attribute + + +class JSONField(forms.CharField): + def clean(self, text): + text = super().clean(text) + + if not self.required and (text is None or text == ""): + return None + + try: + return json.loads(text) + except ValueError: + raise forms.ValidationError("this field must be valid JSON") + + +class DatasetForm(ResourceBaseForm): + class Meta(ResourceBaseForm.Meta): + model = Dataset + exclude = ResourceBaseForm.Meta.exclude + ( + "store", + "styles", + "subtype", + "alternate", + "workspace", + "default_style", + "upload_session", + "resource_type", + "remote_service", + "remote_typename", + "users_geolimits", + "groups_geolimits", + "blob", + "files", + "ows_url", + ) + # widgets = { + # 'title': forms.TextInput({'placeholder': title_help_text}) + # } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["regions"].choices = get_tree_data() + for field in self.fields: + help_text = self.fields[field].help_text + self.fields[field].help_text = None + if help_text != "": + self.fields[field].widget.attrs.update( + { + "class": "has-external-popover", + "data-content": help_text, + "placeholder": help_text, + "data-placement": "right", + "data-container": "body", + "data-html": "true", + } + ) + + +class LayerDescriptionForm(forms.Form): + title = forms.CharField(max_length=300, required=True) + abstract = forms.CharField(max_length=2000, widget=forms.Textarea, required=False) + supplemental_information = forms.CharField(max_length=2000, widget=forms.Textarea, required=False) + data_quality_statement = forms.CharField(max_length=2000, widget=forms.Textarea, required=False) + purpose = forms.CharField(max_length=500, required=False) + keywords = forms.CharField(max_length=500, required=False) + + +class LayerAttributeForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["attribute"].widget.attrs["readonly"] = True + self.fields["display_order"].widget.attrs["size"] = 3 + + class Meta: + model = Attribute + exclude = ( + "attribute_type", + "count", + "min", + "max", + "average", + "median", + "stddev", + "sum", + "unique_values", + "last_stats_updated", + "objects", + ) + + +class DatasetTimeSerieForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + _choises = [(None, "-----")] + [ + (_a.pk, _a.attribute) + for _a in kwargs.get("instance").attributes + if _a.attribute_type in ["xsd:dateTime", "xsd:date"] + ] + self.base_fields.get("attribute").choices = _choises + self.base_fields.get("end_attribute").choices = _choises + super().__init__(*args, **kwargs) + + class Meta: + model = Attribute + fields = ("attribute",) + + attribute = forms.ChoiceField( + required=False, + ) + end_attribute = forms.ChoiceField( + required=False, + ) + presentation = forms.ChoiceField( + required=False, + choices=[ + ("LIST", "List of all the distinct time values"), + ("DISCRETE_INTERVAL", "Intervals defined by the resolution"), + ( + "CONTINUOUS_INTERVAL", + "Continuous Intervals for data that is frequently updated, resolution describes the frequency of updates", + ), + ], + ) + precision_value = forms.IntegerField(required=False) + precision_step = forms.ChoiceField( + required=False, + choices=[("years",) * 2, ("months",) * 2, ("days",) * 2, ("hours",) * 2, ("minutes",) * 2, ("seconds",) * 2], + ) diff --git a/geonode/layers/management/__init__.py b/geonode/layers/management/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/layers/management/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/layers/management/commands/__init__.py b/geonode/layers/management/commands/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/layers/management/commands/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/layers/management/commands/delete_orphaned_layers.py b/geonode/layers/management/commands/delete_orphaned_layers.py new file mode 100644 index 0000000..5458151 --- /dev/null +++ b/geonode/layers/management/commands/delete_orphaned_layers.py @@ -0,0 +1,29 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand + +from geonode.layers.utils import delete_orphaned_datasets + + +class Command(BaseCommand): + help = ("Delete orphaned layers.") + + def handle(self, *args, **options): + delete_orphaned_datasets() diff --git a/geonode/layers/management/commands/layer_notice_types.py b/geonode/layers/management/commands/layer_notice_types.py new file mode 100644 index 0000000..78893cb --- /dev/null +++ b/geonode/layers/management/commands/layer_notice_types.py @@ -0,0 +1,28 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand +from geonode.layers.populate_datasets_data import create_notifications + + +class Command(BaseCommand): + help = ("Create notice types for layers") + + def handle(self, *args, **options): + create_notifications() diff --git a/geonode/layers/management/commands/set_layers_permissions.py b/geonode/layers/management/commands/set_layers_permissions.py new file mode 100644 index 0000000..0e23584 --- /dev/null +++ b/geonode/layers/management/commands/set_layers_permissions.py @@ -0,0 +1,134 @@ +######################################################################### +# +# Copyright (C) 2019 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +from argparse import RawTextHelpFormatter + +from django.core.management.base import BaseCommand +from geonode.layers.models import Dataset +from geonode.layers.utils import set_datasets_permissions + +logger = logging.getLogger("geonode.layers.management.set_layers_permissions") + + +class Command(BaseCommand): + + help = """ + Set/Unset permissions on layers for users and groups. + Arguments: + - users (-u, --users) + - groups (-g, --groups) + - resources (-r, --resources) + - permissions (-p, --permissions) + - delete (-d, --delete) + At least one user or one group is required. + If no resources are typed all the layers will be considered. + At least one permission must be typed. + Multiple inputs can be typed with white space separator. + To unset permissions use the '--delete (-d)' option. + To assign permissions to everyone (anonymous users), you will need to + add the following options: '-u AnonymousUser -g anonymous' + """ + + def create_parser(self, *args, **kwargs): + parser = super().create_parser(*args, **kwargs) + parser.formatter_class = RawTextHelpFormatter + return parser + + def add_arguments(self, parser): + parser.add_argument( + "-r", + "--resources", + dest="resources", + nargs="*", + type=str, + default=[], + help="Resources IDs for which permissions will be assigned to. Default value: [] (all the layers will be considered). " + ) + parser.add_argument( + "-p", + "--permission", + dest="permission", + type=str, + default=None, + help="Permissions to be assigned. " "Allowed values are: view, download, edit and manage.", + ) + parser.add_argument( + "-u", + "--users", + dest="users", + nargs="*", + type=str, + default=[], + help="Users for which permissions will be assigned to. " + "Multiple choices can be typed with comma separator.", + ) + parser.add_argument( + "-g", + "--groups", + dest="groups", + nargs="*", + type=str, + default=[], + help="Groups for which permissions will be assigned to. " + "Multiple choices can be typed with comma separator.", + ) + parser.add_argument( + "-d", + "--delete", + dest="delete_flag", + action="store_true", + default=False, + help="Delete permission if it exists.", + ) + + def handle(self, *args, **options): + # Retrieving the arguments + permissions_name = options.get("permission").replace(" ", "") + + resources_pk = [x.replace(" ", "") for x in options.get("resources", [])] + if resources_pk: + resources_pk = resources_pk[0].split(",") + else: + resources_pk = [x for x in Dataset.objects.values_list('pk', flat=True)] + + users_usernames = [x.replace(" ", "") for x in options.get("users", [])] + if users_usernames: + users_usernames = users_usernames[0].split(",") + + groups_names = [x.replace(" ", "") for x in options.get("groups", [])] + if groups_names: + groups_names = groups_names[0].split(",") + + delete_flag = options.get("delete_flag") + + if isinstance(permissions_name, list): + # it accept one kind of permission per request + raise Exception("Only one permission name must be specified") + + if not users_usernames and not groups_names: + raise Exception("Groups or Usernames must be specified") + + set_datasets_permissions( + permissions_name, + resources_names=resources_pk, + users_usernames=users_usernames, + groups_names=groups_names, + delete_flag=delete_flag + ) diff --git a/geonode/layers/metadata.py b/geonode/layers/metadata.py new file mode 100644 index 0000000..b9357fc --- /dev/null +++ b/geonode/layers/metadata.py @@ -0,0 +1,265 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""Utilities for managing GeoNode resource metadata +""" +# Standard Modules +import logging +import datetime +from owslib.etree import etree as dlxml +from django.conf import settings + +# Geonode functionality +from geonode import GeoNodeException + +# OWSLib functionality +from owslib.csw import CswRecord +from owslib.iso import MD_Metadata, MD_Keywords +from owslib.fgdc import Metadata +from django.utils import timezone + +LOGGER = logging.getLogger(__name__) + + +def set_metadata(xml, identifier="", vals={}, regions=[], keywords=[], custom={}): + """Generate dict of model properties based on XML metadata""" + + # check if document is XML + try: + exml = dlxml.fromstring(xml.encode()) + except Exception as err: + raise GeoNodeException(f"Uploaded XML document is not XML: {str(err)}") + + # check if document is an accepted XML metadata format + tagname = get_tagname(exml) + + if tagname == "GetRecordByIdResponse": # strip CSW element + LOGGER.debug("stripping CSW root element") + exml = list(exml)[0] + tagname = get_tagname(exml) + + if tagname == "MD_Metadata": # ISO + identifier, vals, regions, keywords = iso2dict(exml) + elif tagname == "metadata": # FGDC + identifier, vals, regions, keywords = fgdc2dict(exml) + elif tagname == "Record": # Dublin Core + identifier, vals, regions, keywords = dc2dict(exml) + else: + raise RuntimeError("Unsupported metadata format") + + if not vals.get("date"): + vals["date"] = datetime.datetime.now(timezone.get_current_timezone()).strftime("%Y-%m-%dT%H:%M:%S") + + return [identifier, vals, regions, keywords, custom] + + +def iso2dict(exml): + """generate dict of properties from gmd:MD_Metadata""" + + vals = {} + regions = [] + keywords = [] + + mdata = MD_Metadata(exml) + identifier = mdata.identifier + vals["language"] = mdata.language or mdata.languagecode or "eng" + if mdata.identification[0].spatialrepresentationtype: + vals["spatial_representation_type"] = mdata.identification[0].spatialrepresentationtype[0] + + vals["date"] = sniff_date(mdata.datestamp) + + if hasattr(mdata, "identification"): + vals["title"] = mdata.identification[0].title + vals["abstract"] = mdata.identification[0].abstract + vals["purpose"] = mdata.identification[0].purpose + + if mdata.identification[0].supplementalinformation is not None: + vals["supplemental_information"] = mdata.identification[0].supplementalinformation + + vals["temporal_extent_start"] = getattr(mdata.identification[0], "temporalextent_start", None) + vals["temporal_extent_end"] = getattr(mdata.identification[0], "temporalextent_end", None) + + if len(mdata.identification[0].topiccategory) > 0: + vals["topic_category"] = mdata.identification[0].topiccategory[0] + + if hasattr(mdata.identification[0], "keywords") and len(mdata.identification[0].keywords) > 0: + for kw in mdata.identification[0].keywords: + if kw.type == "place": + regions.extend([_kw.name for _kw in kw.keywords]) + else: + keywords.extend([_kw.name for _kw in kw.keywords]) + + keywords = convert_keyword(keywords, iso2dict=True) + + if len(mdata.identification[0].otherconstraints) > 0: + vals["constraints_other"] = mdata.identification[0].otherconstraints[0] + + vals["purpose"] = mdata.identification[0].purpose + + if mdata.dataquality is not None: + vals["data_quality_statement"] = mdata.dataquality.lineage + + return [identifier, vals, regions, keywords] + + +def fgdc2dict(exml): + """generate dict of properties from FGDC metadata""" + + vals = {} + regions = [] + keywords = [] + + mdata = Metadata(exml) + identifier = mdata.idinfo.datasetid + if hasattr(mdata.idinfo, "citation"): + if hasattr(mdata.idinfo.citation, "citeinfo"): + vals["spatial_representation_type"] = mdata.idinfo.citation.citeinfo["geoform"] + vals["title"] = mdata.idinfo.citation.citeinfo["title"] + + if hasattr(mdata.idinfo, "descript"): + vals["abstract"] = mdata.idinfo.descript.abstract + vals["purpose"] = mdata.idinfo.descript.purpose + if mdata.idinfo.descript.supplinf is not None: + vals["supplemental_information"] = mdata.idinfo.descript.supplinf + + if hasattr(mdata.idinfo, "keywords"): + if mdata.idinfo.keywords.theme: + for theme in mdata.idinfo.keywords.theme: + if theme["themekt"] is not None: + lowered_themekt = theme["themekt"].lower() + + # Owslib doesn't support extracting the Topic Category + # from FGDC. So we add support here. + # http://www.fgdc.gov/metadata/geospatial-metadata-standards + if all(ss in lowered_themekt for ss in ["iso", "19115", "topic"]) and any( + ss in lowered_themekt for ss in ["category", "categories"] + ): + vals["topic_category"] = theme["themekey"][0] + + keywords.extend(theme["themekey"]) + + if mdata.idinfo.keywords.place: + for place in mdata.idinfo.keywords.place: + if "placekey" in place: + regions.extend(place["placekey"]) + + if hasattr(mdata.idinfo.timeperd, "timeinfo"): + if hasattr(mdata.idinfo.timeperd.timeinfo, "rngdates"): + vals["temporal_extent_start"] = sniff_date(mdata.idinfo.timeperd.timeinfo.rngdates.begdate) + vals["temporal_extent_end"] = sniff_date(mdata.idinfo.timeperd.timeinfo.rngdates.enddate) + + vals["constraints_other"] = mdata.idinfo.useconst + raw_date = mdata.metainfo.metd + if raw_date is not None: + vals["date"] = sniff_date(raw_date) + + keywords = convert_keyword(keywords) + + return [identifier, vals, regions, keywords] + + +def dc2dict(exml): + """generate dict of properties from csw:Record""" + + vals = {} + regions = [] + keywords = [] + + mdata = CswRecord(exml) + identifier = mdata.identifier + vals["language"] = mdata.language + vals["spatial_representation_type"] = mdata.type + keywords = mdata.subjects + regions = [mdata.spatial] + vals["temporal_extent_start"] = mdata.temporal + vals["temporal_extent_end"] = mdata.temporal + vals["constraints_other"] = mdata.license + vals["date"] = sniff_date(mdata.modified) + vals["title"] = mdata.title + vals["abstract"] = mdata.abstract + + keywords = convert_keyword(keywords) + + return [identifier, vals, regions, keywords] + + +def sniff_date(datestr): + """ + Attempt to parse date into datetime.datetime object + + Possible inputs: + + '20001122' + '2000-11-22' + '2000-11-22T11:11:11Z' + '2000-11-22T' + """ + + dateformats = ("%Y%m%d", "%Y-%m-%d", "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT", "%Y/%m/%d") + + for dfmt in dateformats: + try: + return datetime.datetime.strptime(datestr.strip(), dfmt) + except (ValueError, AttributeError): + pass + return "" + + +def get_tagname(element): + """get tagname without namespace""" + try: + tagname = element.tag.split("}")[1] + except IndexError: + tagname = element.tag + return tagname + + +def parse_metadata(exml, uuid="", vals={}, regions=[], keywords=[], custom={}): + from django.utils.module_loading import import_string + + available_parsers = settings.METADATA_PARSERS if hasattr(settings, "METADATA_PARSERS") else [] + available_parsers = ["__DEFAULT__"] if len(available_parsers) == 0 else available_parsers + for parser_path in available_parsers: + if parser_path == "__DEFAULT__": + parser_path = "geonode.layers.metadata.set_metadata" + parser = import_string(parser_path) + uuid, vals, regions, keywords, custom = parser(exml, uuid, vals, regions, keywords, custom) + return uuid, vals, regions, keywords, custom + + +def convert_keyword(keywords, iso2dict=False, theme="theme"): + if not iso2dict and keywords: + return [ + { + "keywords": convert_iso_keywords(keywords), + "thesaurus": {"date": None, "datetype": None, "title": None}, + "type": theme, + } + ] + return convert_iso_keywords(keywords) + + +def convert_iso_keywords(keywords): + _keywords = [] + for kw in keywords: + if isinstance(kw, MD_Keywords): + _keywords.append([_kw.name for _kw in kw.keywords]) + else: + _keywords.append(kw) + return _keywords diff --git a/geonode/layers/migrations/0027_auto_20170801_1228.py b/geonode/layers/migrations/0027_auto_20170801_1228.py new file mode 100644 index 0000000..2987a0c --- /dev/null +++ b/geonode/layers/migrations/0027_auto_20170801_1228.py @@ -0,0 +1,31 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '26_to_27'), + ] + + operations = [ + migrations.AlterField( + model_name='layer', + name='abstract_en', + field=models.TextField(help_text='brief narrative summary of the content of the resource(s)', max_length=2000, null=True, verbose_name='abstract', blank=True), + ), + migrations.AlterField( + model_name='layer', + name='data_quality_statement_en', + field=models.TextField(help_text="general explanation of the data producer's knowledge about the lineage of a dataset", max_length=2000, null=True, verbose_name='data quality statement', blank=True), + ), + migrations.AlterField( + model_name='layer', + name='purpose_en', + field=models.TextField(help_text='summary of the intentions with which the resource(s) was developed', max_length=500, null=True, verbose_name='purpose', blank=True), + ), + migrations.AlterField( + model_name='layer', + name='supplemental_information_en', + field=models.TextField(default='No information provided', max_length=2000, null=True, verbose_name='supplemental information', help_text='any other descriptive information about the dataset'), + ), + ] diff --git a/geonode/layers/migrations/0027_auto_20170801_1228_squashed_0033_auto_20180606_1543.py b/geonode/layers/migrations/0027_auto_20170801_1228_squashed_0033_auto_20180606_1543.py new file mode 100644 index 0000000..a287ceb --- /dev/null +++ b/geonode/layers/migrations/0027_auto_20170801_1228_squashed_0033_auto_20180606_1543.py @@ -0,0 +1,109 @@ +# Generated by Django 1.11.20 on 2019-04-04 08:27 + +import django.core.files.storage +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [('layers', '0027_auto_20170801_1228'), ('layers', '0028_auto_20171218_0249'), ('layers', '0029_layer_service'), ('layers', '0030_auto_20180228_0812'), ('layers', '0031_uploadsession_resource'), ('layers', '0032_auto_20180424_1638'), ('layers', '0033_auto_20180606_1543')] + + dependencies = [ + ('layers', '26_to_27'), + ('base', '0033_auto_20180330_0951'), + ('services', '0030_auto_20171212_0518'), + ] + + operations = [ + migrations.AlterField( + model_name='layer', + name='abstract_en', + field=models.TextField(blank=True, help_text='brief narrative summary of the content of the resource(s)', max_length=2000, null=True, verbose_name='abstract'), + ), + migrations.AlterField( + model_name='layer', + name='data_quality_statement_en', + field=models.TextField(blank=True, help_text="general explanation of the data producer's knowledge about the lineage of a dataset", max_length=2000, null=True, verbose_name='data quality statement'), + ), + migrations.AlterField( + model_name='layer', + name='purpose_en', + field=models.TextField(blank=True, help_text='summary of the intentions with which the resource(s) was developed', max_length=500, null=True, verbose_name='purpose'), + ), + migrations.AlterField( + model_name='layer', + name='supplemental_information_en', + field=models.TextField(default='No information provided', help_text='any other descriptive information about the dataset', max_length=2000, null=True, verbose_name='supplemental information'), + ), + migrations.AlterField( + model_name='layer', + name='default_style', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='layer_default_style', to='layers.Style'), + ), + migrations.AddField( + model_name='layer', + name='remote_service', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='services.Service'), + ), + migrations.AddField( + model_name='uploadsession', + name='resource', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase'), + ), + migrations.AlterField( + model_name='layerfile', + name='file', + field=models.FileField(max_length=255, storage=django.core.files.storage.FileSystemStorage(base_url='/uploaded/'), upload_to='layers/%Y/%m/%d'), + ), + migrations.AlterField( + model_name='layer', + name='elevation_regex', + field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Elevation regex'), + ), + migrations.AlterField( + model_name='layer', + name='has_elevation', + field=models.BooleanField(default=False, verbose_name='Has elevation?'), + ), + migrations.AlterField( + model_name='layer', + name='has_time', + field=models.BooleanField(default=False, verbose_name='Has time?'), + ), + migrations.AlterField( + model_name='layer', + name='is_mosaic', + field=models.BooleanField(default=False, verbose_name='Is mosaic?'), + ), + migrations.AlterField( + model_name='layer', + name='name', + field=models.CharField(max_length=128, verbose_name='Name'), + ), + migrations.AlterField( + model_name='layer', + name='store', + field=models.CharField(max_length=128, verbose_name='Store'), + ), + migrations.AlterField( + model_name='layer', + name='storeType', + field=models.CharField(max_length=128, verbose_name='Storetype'), + ), + migrations.AlterField( + model_name='layer', + name='time_regex', + field=models.CharField(blank=True, choices=[('[0-9]{8}', 'YYYYMMDD'), ('[0-9]{8}T[0-9]{6}', "YYYYMMDD'T'hhmmss"), ('[0-9]{8}T[0-9]{6}Z', "YYYYMMDD'T'hhmmss'Z'")], max_length=128, null=True, verbose_name='Time regex'), + ), + migrations.AlterField( + model_name='layer', + name='typename', + field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Typename'), + ), + migrations.AlterField( + model_name='layer', + name='workspace', + field=models.CharField(max_length=128, verbose_name='Workspace'), + ), + ] diff --git a/geonode/layers/migrations/0028_auto_20171218_0249.py b/geonode/layers/migrations/0028_auto_20171218_0249.py new file mode 100644 index 0000000..d70d401 --- /dev/null +++ b/geonode/layers/migrations/0028_auto_20171218_0249.py @@ -0,0 +1,17 @@ +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0027_auto_20170801_1228'), + ] + + operations = [ + migrations.AlterField( + model_name='layer', + name='default_style', + field=models.ForeignKey(related_name='layer_default_style', on_delete=django.db.models.deletion.SET_NULL, blank=True, to='layers.Style', null=True), + ), + ] diff --git a/geonode/layers/migrations/0028_auto_20200323_1945.py b/geonode/layers/migrations/0028_auto_20200323_1945.py new file mode 100644 index 0000000..c15f64c --- /dev/null +++ b/geonode/layers/migrations/0028_auto_20200323_1945.py @@ -0,0 +1,19 @@ +# Populating polygons into bbox_polygon field + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0044_resourcebase_bbox_polygon'), + ('layers', '0027_auto_20170801_1228_squashed_0033_auto_20180606_1543'), + ] + + # This migration used to populate bbox_polygon field for all Layers. + # Now all resources are updated in base/0044 + # See #12144 + + operations = [ + migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), + ] diff --git a/geonode/layers/migrations/0028_auto_20200610_0759.py b/geonode/layers/migrations/0028_auto_20200610_0759.py new file mode 100644 index 0000000..5d310c4 --- /dev/null +++ b/geonode/layers/migrations/0028_auto_20200610_0759.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.13 on 2020-06-10 07:59 + +from django.db import migrations, models +import tinymce.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0027_auto_20170801_1228_squashed_0033_auto_20180606_1543'), + ] + + operations = [ + migrations.AddField( + model_name='attribute', + name='featureinfo_type', + field=models.CharField(choices=[('type_property', 'Property-Label'), ('type_href', 'HREF-Link'), ('type_image', 'Image')], default='type_property', help_text='specifies if the attribute should be rendered with an HTML widget on GetFeatureInfo template.', max_length=255, verbose_name='featureinfo type'), + ), + migrations.AddField( + model_name='layer', + name='featureinfo_custom_template', + field=tinymce.models.HTMLField(blank=True, help_text='the custom GetFeatureInfo template HTML contents.', null=True, verbose_name='featureinfo custom template'), + ), + migrations.AddField( + model_name='layer', + name='use_featureinfo_custom_template', + field=models.BooleanField(default=False, help_text='specifies wether or not use a custom GetFeatureInfo template.', verbose_name='use featureinfo custom template?'), + ), + ] diff --git a/geonode/layers/migrations/0029_auto_20200610_0803.py b/geonode/layers/migrations/0029_auto_20200610_0803.py new file mode 100644 index 0000000..79eab85 --- /dev/null +++ b/geonode/layers/migrations/0029_auto_20200610_0803.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.13 on 2020-06-10 08:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0028_auto_20200610_0759'), + ] + + operations = [ + migrations.AlterField( + model_name='attribute', + name='featureinfo_type', + field=models.CharField(choices=[('type_property', 'Property-Label'), ('type_href', 'HREF-Link'), ('type_image', 'Image'), ('type_video', 'Video'), ('type_audio', 'Audio')], default='type_property', help_text='specifies if the attribute should be rendered with an HTML widget on GetFeatureInfo template.', max_length=255, verbose_name='featureinfo type'), + ), + ] diff --git a/geonode/layers/migrations/0029_layer_service.py b/geonode/layers/migrations/0029_layer_service.py new file mode 100644 index 0000000..038aafa --- /dev/null +++ b/geonode/layers/migrations/0029_layer_service.py @@ -0,0 +1,17 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('services', '0030_auto_20171212_0518'), + ('layers', '0028_auto_20171218_0249'), + ] + + operations = [ + migrations.AddField( + model_name='layer', + name='service', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='services.Service', null=True), + ), + ] diff --git a/geonode/layers/migrations/0030_auto_20180228_0812.py b/geonode/layers/migrations/0030_auto_20180228_0812.py new file mode 100644 index 0000000..a06c15c --- /dev/null +++ b/geonode/layers/migrations/0030_auto_20180228_0812.py @@ -0,0 +1,24 @@ +# Generated by Django 1.9.13 on 2018-02-28 14:12 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('services', '0030_auto_20171212_0518'), + ('layers', '0029_layer_service'), + ] + + operations = [ + migrations.RemoveField( + model_name='layer', + name='service', + ), + migrations.AddField( + model_name='layer', + name='remote_service', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='services.Service'), + ), + ] diff --git a/geonode/layers/migrations/0030_auto_20200610_0825.py b/geonode/layers/migrations/0030_auto_20200610_0825.py new file mode 100644 index 0000000..ca7740f --- /dev/null +++ b/geonode/layers/migrations/0030_auto_20200610_0825.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.13 on 2020-06-10 08:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0029_auto_20200610_0803'), + ] + + operations = [ + migrations.AlterField( + model_name='attribute', + name='featureinfo_type', + field=models.CharField(choices=[('type_property', 'Property-Label'), ('type_href', 'HREF-Link'), ('type_image', 'Image'), ('type_video', 'Video'), ('type_audio', 'Audio'), ('type_iframe', 'iFrame')], default='type_property', help_text='specifies if the attribute should be rendered with an HTML widget on GetFeatureInfo template.', max_length=255, verbose_name='featureinfo type'), + ), + ] diff --git a/geonode/layers/migrations/0031_auto_20200612_1533.py b/geonode/layers/migrations/0031_auto_20200612_1533.py new file mode 100644 index 0000000..4f3706a --- /dev/null +++ b/geonode/layers/migrations/0031_auto_20200612_1533.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.13 on 2020-06-12 15:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0030_auto_20200610_0825'), + ] + + operations = [ + migrations.AlterField( + model_name='attribute', + name='featureinfo_type', + field=models.CharField(choices=[('type_property', 'Label'), ('type_href', 'URL'), ('type_image', 'Image'), ('type_video_mp4', 'Video (mp4)'), ('type_video_ogg', 'Video (ogg)'), ('type_video_youtube', 'Video (YouTube - embed)'), ('type_audio', 'Audio'), ('type_iframe', 'IFRAME')], default='type_property', help_text='specifies if the attribute should be rendered with an HTML widget on GetFeatureInfo template.', max_length=255, verbose_name='featureinfo type'), + ), + ] diff --git a/geonode/layers/migrations/0031_uploadsession_resource.py b/geonode/layers/migrations/0031_uploadsession_resource.py new file mode 100644 index 0000000..40f746e --- /dev/null +++ b/geonode/layers/migrations/0031_uploadsession_resource.py @@ -0,0 +1,20 @@ +# Generated by Django 1.11.11 on 2018-04-24 16:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0033_auto_20180330_0951'), + ('layers', '0030_auto_20180228_0812'), + ] + + operations = [ + migrations.AddField( + model_name='uploadsession', + name='resource', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase'), + ), + ] diff --git a/geonode/layers/migrations/0032_auto_20180424_1638.py b/geonode/layers/migrations/0032_auto_20180424_1638.py new file mode 100644 index 0000000..7197057 --- /dev/null +++ b/geonode/layers/migrations/0032_auto_20180424_1638.py @@ -0,0 +1,19 @@ +# Generated by Django 1.11.11 on 2018-04-24 16:38 + +import django.core.files.storage +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0031_uploadsession_resource'), + ] + + operations = [ + migrations.AlterField( + model_name='layerfile', + name='file', + field=models.FileField(max_length=255, storage=django.core.files.storage.FileSystemStorage(base_url='/uploaded/'), upload_to='layers/%Y/%m/%d'), + ), + ] diff --git a/geonode/layers/migrations/0032_auto_20200612_1544.py b/geonode/layers/migrations/0032_auto_20200612_1544.py new file mode 100644 index 0000000..a57f8f2 --- /dev/null +++ b/geonode/layers/migrations/0032_auto_20200612_1544.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.13 on 2020-06-12 15:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0031_auto_20200612_1533'), + ] + + operations = [ + migrations.AlterField( + model_name='attribute', + name='featureinfo_type', + field=models.CharField(choices=[('type_property', 'Label'), ('type_href', 'URL'), ('type_image', 'Image'), ('type_video_mp4', 'Video (mp4)'), ('type_video_ogg', 'Video (ogg)'), ('type_video_webm', 'Video (webm)'), ('type_video_3gp', 'Video (3gp)'), ('type_video_flv', 'Video (flv)'), ('type_video_youtube', 'Video (YouTube/VIMEO - embedded)'), ('type_audio', 'Audio'), ('type_iframe', 'IFRAME')], default='type_property', help_text='specifies if the attribute should be rendered with an HTML widget on GetFeatureInfo template.', max_length=255, verbose_name='featureinfo type'), + ), + ] diff --git a/geonode/layers/migrations/0033_auto_20180606_1543.py b/geonode/layers/migrations/0033_auto_20180606_1543.py new file mode 100644 index 0000000..8f7613d --- /dev/null +++ b/geonode/layers/migrations/0033_auto_20180606_1543.py @@ -0,0 +1,63 @@ +# Generated by Django 1.11.11 on 2018-06-06 15:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0032_auto_20180424_1638'), + ] + + operations = [ + migrations.AlterField( + model_name='layer', + name='elevation_regex', + field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Elevation regex'), + ), + migrations.AlterField( + model_name='layer', + name='has_elevation', + field=models.BooleanField(default=False, verbose_name='Has elevation?'), + ), + migrations.AlterField( + model_name='layer', + name='has_time', + field=models.BooleanField(default=False, verbose_name='Has time?'), + ), + migrations.AlterField( + model_name='layer', + name='is_mosaic', + field=models.BooleanField(default=False, verbose_name='Is mosaic?'), + ), + migrations.AlterField( + model_name='layer', + name='name', + field=models.CharField(max_length=128, verbose_name='Name'), + ), + migrations.AlterField( + model_name='layer', + name='store', + field=models.CharField(max_length=128, verbose_name='Store'), + ), + migrations.AlterField( + model_name='layer', + name='storeType', + field=models.CharField(max_length=128, verbose_name='Storetype'), + ), + migrations.AlterField( + model_name='layer', + name='time_regex', + field=models.CharField(blank=True, choices=[('[0-9]{8}', 'YYYYMMDD'), ('[0-9]{8}T[0-9]{6}', "YYYYMMDD'T'hhmmss"), ('[0-9]{8}T[0-9]{6}Z', "YYYYMMDD'T'hhmmss'Z'")], max_length=128, null=True, verbose_name='Time regex'), + ), + migrations.AlterField( + model_name='layer', + name='typename', + field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Typename'), + ), + migrations.AlterField( + model_name='layer', + name='workspace', + field=models.CharField(max_length=128, verbose_name='Workspace'), + ), + ] diff --git a/geonode/layers/migrations/0033_merge_20200618_2150.py b/geonode/layers/migrations/0033_merge_20200618_2150.py new file mode 100644 index 0000000..fef353c --- /dev/null +++ b/geonode/layers/migrations/0033_merge_20200618_2150.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.13 on 2020-06-18 21:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0028_auto_20200323_1945'), + ('layers', '0032_auto_20200612_1544'), + ] + + operations = [ + ] diff --git a/geonode/layers/migrations/0034_auto_20210329_1458.py b/geonode/layers/migrations/0034_auto_20210329_1458.py new file mode 100644 index 0000000..78aab1b --- /dev/null +++ b/geonode/layers/migrations/0034_auto_20210329_1458.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.16 on 2021-03-29 14:58 + +import django.core.files.storage +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0033_merge_20200618_2150'), + ] + + operations = [ + migrations.AlterField( + model_name='layerfile', + name='file', + field=models.FileField(max_length=4096, storage=django.core.files.storage.FileSystemStorage(base_url='/uploaded/'), upload_to='layers/%Y/%m/%d'), + ), + migrations.AlterField( + model_name='layerfile', + name='name', + field=models.CharField(max_length=4096), + ), + ] diff --git a/geonode/layers/migrations/0035_auto_20210525_0847.py b/geonode/layers/migrations/0035_auto_20210525_0847.py new file mode 100644 index 0000000..52b3b41 --- /dev/null +++ b/geonode/layers/migrations/0035_auto_20210525_0847.py @@ -0,0 +1,31 @@ +# Generated by Django 2.2.20 on 2021-05-25 08:47 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0034_auto_20210329_1458'), + ] + + operations = [ + migrations.RemoveField( + model_name='uploadsession', + name='resource', + ), + migrations.RemoveField( + model_name='uploadsession', + name='user', + ), + migrations.RemoveField( + model_name='layer', + name='upload_session', + ), + migrations.DeleteModel( + name='LayerFile', + ), + migrations.DeleteModel( + name='UploadSession', + ), + ] diff --git a/geonode/layers/migrations/0036_remove_layer_storetype.py b/geonode/layers/migrations/0036_remove_layer_storetype.py new file mode 100644 index 0000000..be05feb --- /dev/null +++ b/geonode/layers/migrations/0036_remove_layer_storetype.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-07-06 13:01 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0035_auto_20210525_0847'), + ('base', '0067_resourcebase_storetype'), + ] + + operations = [ + migrations.RemoveField( + model_name='layer', + name='storeType', + ), + ] diff --git a/geonode/layers/migrations/0037_layer_ptype.py b/geonode/layers/migrations/0037_layer_ptype.py new file mode 100644 index 0000000..0a0725c --- /dev/null +++ b/geonode/layers/migrations/0037_layer_ptype.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.4 on 2021-07-20 17:09 + +from django.db import migrations, models +from geonode.layers.enumerations import GXP_PTYPES + + +def update_remotes_attributes(apps, schema_editor): + """Updating Remote Services related attributes + + - For the time being we need to iterate over the layers since the remote service properties + are built at runtime and therefore we cannot benefit of an optimized Subquery. + """ + MyModel = apps.get_model('layers', 'Layer') + FkModel = apps.get_model('services', 'Service') + + for _m in MyModel.objects.filter(models.Q(remote_service__isnull=False)): + service = FkModel.objects.get(pk=_m.remote_service) + MyModel.objects.filter(pk=_m).update( + sourcetype='REMOTE', + remote_typename=service.name, + ptype=GXP_PTYPES.get(service.type, None)) + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0036_remove_layer_storetype'), + ('base', '0070_auto_20210720_1709') + ] + + operations = [ + migrations.AddField( + model_name='layer', + name='ptype', + field=models.CharField(default='gxp_wmscsource', max_length=80, verbose_name='P-Type'), + ), + migrations.RunPython(update_remotes_attributes, migrations.RunPython.noop) + ] diff --git a/geonode/layers/migrations/0038_rename_layer_dataset.py b/geonode/layers/migrations/0038_rename_layer_dataset.py new file mode 100644 index 0000000..5d9e70f --- /dev/null +++ b/geonode/layers/migrations/0038_rename_layer_dataset.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.4 on 2021-07-20 11:05 + +from django.conf import settings +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('services', '0045_auto_20210629_1355'), + ('upload', '0033_auto_20210531_1252'), + ('auth', '0012_alter_user_first_name_max_length'), + ('harvesting', '0026_harvestableresource_last_harvesting_succeeded'), + ('base', '0068_rename_storetype_resourcebase_subtype'), + ('contenttypes', '0002_remove_content_type_name'), + ('layers', '0037_layer_ptype'), + ] + + operations = [ + migrations.RenameModel( + old_name='Layer', + new_name='Dataset', + ), + ] diff --git a/geonode/layers/migrations/0039_auto_20210720_1311.py b/geonode/layers/migrations/0039_auto_20210720_1311.py new file mode 100644 index 0000000..0ba668d --- /dev/null +++ b/geonode/layers/migrations/0039_auto_20210720_1311.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.4 on 2021-07-20 13:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0038_rename_layer_dataset'), + ] + + operations = [ + migrations.AlterModelOptions( + name='dataset', + options={'permissions': (('change_dataset_data', 'Can edit layer data'), ('change_dataset_style', 'Can change layer style'))}, + ), + migrations.RenameField( + model_name='attribute', + old_name='layer', + new_name='dataset', + ), + migrations.AlterField( + model_name='dataset', + name='default_style', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_default_style', to='layers.style'), + ), + migrations.AlterField( + model_name='dataset', + name='styles', + field=models.ManyToManyField(related_name='dataset_styles', to='layers.Style'), + ), + ] diff --git a/geonode/layers/migrations/0040_dataset_ows_url.py b/geonode/layers/migrations/0040_dataset_ows_url.py new file mode 100644 index 0000000..e28f074 --- /dev/null +++ b/geonode/layers/migrations/0040_dataset_ows_url.py @@ -0,0 +1,49 @@ +# Generated by Django 3.2.4 on 2021-07-21 15:29 + +from django.urls import reverse +from django.conf import settings +from urllib.parse import urljoin +from django.db import migrations, models + +from geonode.utils import OGC_Servers_Handler +from geonode.services.enumerations import INDEXED + +ogc_settings = OGC_Servers_Handler(settings.OGC_SERVER)['default'] + + +def update_ows_url(apps, schema_editor): + """Updating Layer 'ows_url' if referencing an INDEXED Remote Service + """ + MyModel = apps.get_model('layers', 'Dataset') + FkModel = apps.get_model('services', 'Service') + + def _lambda(x): return x[1] if not x[2] else urljoin(settings.SITEURL, reverse('service_proxy', args=[x[0]])) + + for _m in MyModel.objects.filter(models.Q(remote_service__isnull=False) & models.Q(remote_service__method=INDEXED)): + service_url_query = FkModel.objects.filter(pk=_m.remote_service).values_list('id', 'base_url', 'proxy_base')[:1] + MyModel.objects.filter(pk=_m).update( + ows_url=_lambda(list(service_url_query)[0]) + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0039_auto_20210720_1311'), + ('base', '0071_remove_resourcebase_ows_url'), + ] + + operations = [ + migrations.AddField( + model_name='dataset', + name='ows_url', + field=models.URLField(blank=True, default=urljoin(ogc_settings.PUBLIC_LOCATION, 'ows'), + help_text='The URL of the OWS service providing this layer, if any exists.', null=True, verbose_name='ows URL'), + ), + migrations.RunPython(update_ows_url, migrations.RunPython.noop), + migrations.AlterField( + model_name='dataset', + name='ows_url', + field=models.URLField(blank=True, help_text='The URL of the OWS service providing this layer, if any exists.', null=True, verbose_name='ows URL'), + ), + ] diff --git a/geonode/layers/migrations/0041_uodate_resource_type_field.py b/geonode/layers/migrations/0041_uodate_resource_type_field.py new file mode 100644 index 0000000..bd74668 --- /dev/null +++ b/geonode/layers/migrations/0041_uodate_resource_type_field.py @@ -0,0 +1,24 @@ +from django.db import migrations + + +def update_resource_type(apps, schema_editor): + """Updating Layer 'resource_type' from 'layer' to 'dataset' + """ + MyModel = apps.get_model('layers', 'Dataset') + _models = MyModel.objects.filter(resource_type='layer') + for _m in _models: + _m.resource_type='dataset' + + MyModel.objects.bulk_update(_models, ['resource_type']) + + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0040_dataset_ows_url'), + ] + + operations = [ + migrations.RunPython(update_resource_type, migrations.RunPython.noop), + ] diff --git a/geonode/layers/migrations/0042_alter_dataset_options.py b/geonode/layers/migrations/0042_alter_dataset_options.py new file mode 100644 index 0000000..16292b0 --- /dev/null +++ b/geonode/layers/migrations/0042_alter_dataset_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-07-27 11:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0041_uodate_resource_type_field'), + ] + + operations = [ + migrations.AlterModelOptions( + name='dataset', + options={'base_manager_name': 'objects', 'permissions': (('change_dataset_data', 'Can edit layer data'), ('change_dataset_style', 'Can change layer style'))}, + ), + ] diff --git a/geonode/layers/migrations/0043_auto_20220225_1605.py b/geonode/layers/migrations/0043_auto_20220225_1605.py new file mode 100644 index 0000000..4a14bbb --- /dev/null +++ b/geonode/layers/migrations/0043_auto_20220225_1605.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.12 on 2022-02-25 16:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0042_alter_dataset_options'), + ] + + operations = [ + migrations.AlterField( + model_name='dataset', + name='name', + field=models.CharField(max_length=255, verbose_name='Name'), + ), + migrations.AlterField( + model_name='dataset', + name='ptype', + field=models.CharField(default='gxp_wmscsource', max_length=255, verbose_name='P-Type'), + ), + migrations.AlterField( + model_name='dataset', + name='store', + field=models.CharField(max_length=255, verbose_name='Store'), + ), + migrations.AlterField( + model_name='dataset', + name='typename', + field=models.CharField(blank=True, max_length=255, null=True, verbose_name='Typename'), + ), + migrations.AlterField( + model_name='dataset', + name='workspace', + field=models.CharField(max_length=255, verbose_name='Workspace'), + ), + ] diff --git a/geonode/layers/migrations/0044_alter_dataset_unique_together.py b/geonode/layers/migrations/0044_alter_dataset_unique_together.py new file mode 100644 index 0000000..cea4008 --- /dev/null +++ b/geonode/layers/migrations/0044_alter_dataset_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.12 on 2022-04-11 22:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0043_auto_20220225_1605'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='dataset', + unique_together={('store', 'workspace', 'name')}, + ), + ] diff --git a/geonode/layers/migrations/24_initial.py b/geonode/layers/migrations/24_initial.py new file mode 100644 index 0000000..5744398 --- /dev/null +++ b/geonode/layers/migrations/24_initial.py @@ -0,0 +1,120 @@ +from django.db import migrations, models +import datetime +from django.conf import settings +from django.utils.timezone import now +import django.core.files.storage + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('base', '__first__'), + ] + + operations = [ + migrations.CreateModel( + name='Attribute', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('attribute', models.CharField(help_text='name of attribute as stored in shapefile/spatial database', max_length=255, null=True, verbose_name='attribute name')), + ('description', models.CharField(help_text='description of attribute to be used in metadata', max_length=255, null=True, verbose_name='attribute description', blank=True)), + ('attribute_label', models.CharField(help_text='title of attribute as displayed in GeoNode', max_length=255, null=True, verbose_name='attribute label', blank=True)), + ('attribute_type', models.CharField(default='xsd:string', help_text='the data type of the attribute (integer, string, geometry, etc)', max_length=50, verbose_name='attribute type')), + ('visible', models.BooleanField(default=True, help_text='specifies if the attribute should be displayed in identify results', verbose_name='visible?')), + ('display_order', models.IntegerField(default=1, help_text='specifies the order in which attribute should be displayed in identify results', verbose_name='display order')), + ('count', models.IntegerField(default=1, help_text='count value for this field', verbose_name='count')), + ('min', models.CharField(default='NA', max_length=255, null=True, verbose_name='min', help_text='minimum value for this field')), + ('max', models.CharField(default='NA', max_length=255, null=True, verbose_name='max', help_text='maximum value for this field')), + ('average', models.CharField(default='NA', max_length=255, null=True, verbose_name='average', help_text='average value for this field')), + ('median', models.CharField(default='NA', max_length=255, null=True, verbose_name='median', help_text='median value for this field')), + ('stddev', models.CharField(default='NA', max_length=255, null=True, verbose_name='standard deviation', help_text='standard deviation for this field')), + ('sum', models.CharField(default='NA', max_length=255, null=True, verbose_name='sum', help_text='sum value for this field')), + ('unique_values', models.TextField(default='NA', null=True, verbose_name='unique values for this field', blank=True)), + ('last_stats_updated', models.DateTimeField(default=now, help_text='date when attribute statistics were last updated', verbose_name='last modified')), + ], + ), + migrations.CreateModel( + name='Layer', + fields=[ + ('resourcebase_ptr', models.OneToOneField(parent_link=True, on_delete=models.CASCADE, + auto_created=True, primary_key=True, serialize=False, to='base.ResourceBase')), + ('title_en', models.CharField(help_text='name by which the cited resource is known', max_length=255, null=True, verbose_name='title')), + ('abstract_en', models.TextField(help_text='brief narrative summary of the content of the resource(s)', null=True, verbose_name='abstract', blank=True)), + ('purpose_en', models.TextField(help_text='summary of the intentions with which the resource(s) was developed', null=True, verbose_name='purpose', blank=True)), + ('constraints_other_en', models.TextField(help_text='other restrictions and legal prerequisites for accessing and using the resource or metadata', null=True, verbose_name='restrictions other', blank=True)), + ('supplemental_information_en', models.TextField(default='No information provided', help_text='any other descriptive information about the dataset', null=True, verbose_name='supplemental information')), + ('data_quality_statement_en', models.TextField(help_text="general explanation of the data producer's knowledge about the lineage of a dataset", null=True, verbose_name='data quality statement', blank=True)), + ('workspace', models.CharField(max_length=128)), + ('store', models.CharField(max_length=128)), + ('storeType', models.CharField(max_length=128)), + ('name', models.CharField(max_length=128)), + ('typename', models.CharField(max_length=128, null=True, blank=True)), + ('charset', models.CharField(default='UTF-8', max_length=255)), + ], + options={ + 'permissions': (('change_layer_data', 'Can edit layer data'), ('change_layer_style', 'Can change layer style')), + }, + bases=('base.resourcebase',), + ), + migrations.CreateModel( + name='LayerFile', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('base', models.BooleanField(default=False)), + ('file', models.FileField(storage=django.core.files.storage.FileSystemStorage(base_url='/uploaded/'), max_length=255, upload_to='layers')), + ], + ), + migrations.CreateModel( + name='Style', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(unique=True, max_length=255, verbose_name='style name')), + ('sld_title', models.CharField(max_length=255, null=True, blank=True)), + ('sld_body', models.TextField(null=True, verbose_name='sld text', blank=True)), + ('sld_version', models.CharField(max_length=12, null=True, verbose_name='sld version', blank=True)), + ('sld_url', models.CharField(max_length=1000, null=True, verbose_name='sld url')), + ('workspace', models.CharField(max_length=255, null=True, blank=True)), + ], + ), + migrations.CreateModel( + name='UploadSession', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('date', models.DateTimeField(auto_now=True)), + ('processed', models.BooleanField(default=False)), + ('error', models.TextField(null=True, blank=True)), + ('traceback', models.TextField(null=True, blank=True)), + ('context', models.TextField(null=True, blank=True)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), + ], + ), + + migrations.AddField( + model_name='layerfile', + name='upload_session', + field=models.ForeignKey(to='layers.UploadSession', on_delete=models.CASCADE), + ), + migrations.AddField( + model_name='layer', + name='default_style', + field=models.ForeignKey(related_name='layer_default_style', blank=True, + on_delete=models.CASCADE, to='layers.Style', null=True), + ), + migrations.AddField( + model_name='layer', + name='styles', + field=models.ManyToManyField(related_name='layer_styles', to='layers.Style'), + ), + migrations.AddField( + model_name='layer', + name='upload_session', + field=models.ForeignKey(blank=True, on_delete=models.CASCADE, to='layers.UploadSession', null=True), + ), + migrations.AddField( + model_name='attribute', + name='layer', + field=models.ForeignKey(related_name='attribute_set', on_delete=models.CASCADE, to='layers.Layer'), + ), + ] diff --git a/geonode/layers/migrations/24_to_26.py b/geonode/layers/migrations/24_to_26.py new file mode 100644 index 0000000..0f7f8a9 --- /dev/null +++ b/geonode/layers/migrations/24_to_26.py @@ -0,0 +1,36 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '24_initial'), + ] + + operations = [ + migrations.AddField( + model_name='layer', + name='elevation_regex', + field=models.CharField(max_length=128, null=True, blank=True), + ), + migrations.AddField( + model_name='layer', + name='has_elevation', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='layer', + name='has_time', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='layer', + name='is_mosaic', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='layer', + name='time_regex', + field=models.CharField(blank=True, max_length=128, null=True, choices=[('[0-9]{8}', 'YYYYMMDD'), ('[0-9]{8}T[0-9]{6}', "YYYYMMDD'T'hhmmss"), ('[0-9]{8}T[0-9]{6}Z', "YYYYMMDD'T'hhmmss'Z'")]), + ), + ] diff --git a/geonode/layers/migrations/26_to_27.py b/geonode/layers/migrations/26_to_27.py new file mode 100644 index 0000000..44d27ab --- /dev/null +++ b/geonode/layers/migrations/26_to_27.py @@ -0,0 +1,19 @@ +from django.db import migrations, models +from django.db.models import F + +def copy_typename(apps, schema_editor): + Layer = apps.get_model('layers', 'layer') + ResourceBase = apps.get_model('base', 'resourcebase') + for row in Layer.objects.all(): + row.alternate = row.typename + row.save(update_fields=['alternate']) + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '24_to_26'), + ] + + operations = [ + migrations.RunPython(copy_typename, migrations.RunPython.noop), + ] diff --git a/geonode/layers/migrations/__init__.py b/geonode/layers/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geonode/layers/models.py b/geonode/layers/models.py new file mode 100644 index 0000000..016597d --- /dev/null +++ b/geonode/layers/models.py @@ -0,0 +1,593 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import itertools +import re +import logging + +from django.conf import settings +from django.db import models +from django.urls import reverse +from django.utils.timezone import now +from django.utils.functional import classproperty +from django.utils.translation import gettext_lazy as _ + +from tinymce.models import HTMLField + +from geonode.client.hooks import hookset +from geonode.utils import build_absolute_uri, check_shp_columnnames +from geonode.security.models import PermissionLevelMixin +from geonode.groups.conf import settings as groups_settings +from geonode.security.permissions import ( + VIEW_PERMISSIONS, + OWNER_PERMISSIONS, + DOWNLOAD_PERMISSIONS, + DATASET_ADMIN_PERMISSIONS, +) +from geonode.base.models import ResourceBase, ResourceBaseManager, LinkedResource + +logger = logging.getLogger("geonode.layers.models") + +shp_exts = [ + ".shp", +] +csv_exts = [".csv"] +kml_exts = [".kml"] +vec_exts = shp_exts + csv_exts + kml_exts +cov_exts = [".tif", ".tiff", ".geotiff", ".geotif", ".asc"] + +TIME_REGEX = ( + ("[0-9]{8}", _("YYYYMMDD")), + ("[0-9]{8}T[0-9]{6}", _("YYYYMMDD'T'hhmmss")), + ("[0-9]{8}T[0-9]{6}Z", _("YYYYMMDD'T'hhmmss'Z'")), +) + +TIME_REGEX_FORMAT = {"[0-9]{8}": "%Y%m%d", "[0-9]{8}T[0-9]{6}": "%Y%m%dT%H%M%S", "[0-9]{8}T[0-9]{6}Z": "%Y%m%dT%H%M%SZ"} + + +class Style(models.Model, PermissionLevelMixin): + """Model for storing styles.""" + + name = models.CharField(_("style name"), max_length=255, unique=True) + sld_title = models.CharField(max_length=255, null=True, blank=True) + sld_body = models.TextField(_("sld text"), null=True, blank=True) + sld_version = models.CharField(_("sld version"), max_length=12, null=True, blank=True) + sld_url = models.CharField(_("sld url"), null=True, max_length=1000) + workspace = models.CharField(max_length=255, null=True, blank=True) + + def __str__(self): + return str(self.name) + + def absolute_url(self): + if self.sld_url: + if self.sld_url.startswith(settings.OGC_SERVER["default"]["LOCATION"]): + return self.sld_url.split(settings.OGC_SERVER["default"]["LOCATION"], 1)[1] + elif self.sld_url.startswith(settings.OGC_SERVER["default"]["PUBLIC_LOCATION"]): + return self.sld_url.split(settings.OGC_SERVER["default"]["PUBLIC_LOCATION"], 1)[1] + + return self.sld_url + else: + logger.error(f"SLD URL is empty for Style {self.name}") + return None + + def get_self_resource(self): + """Get associated resource base.""" + # Associate this model with resource + try: + dataset = self.dataset_styles.first() + """:type: Dataset""" + return dataset.get_self_resource() + except Exception: + return None + + +class DatasetManager(ResourceBaseManager): + def __init__(self): + models.Manager.__init__(self) + + +class Dataset(ResourceBase): + """ + Dataset (inherits ResourceBase fields) + """ + + PERMISSIONS = { + "write": [ + "change_dataset_data", + "change_dataset_style", + ] + } + + # internal fields + objects = DatasetManager() + workspace = models.CharField(_("Workspace"), max_length=255) + store = models.CharField(_("Store"), max_length=255) + name = models.CharField(_("Name"), max_length=255) + typename = models.CharField(_("Typename"), max_length=255, null=True, blank=True) + ows_url = models.URLField( + _("ows URL"), + null=True, + blank=True, + help_text=_("The URL of the OWS service providing this layer, if any exists."), + ) + + is_mosaic = models.BooleanField(_("Is mosaic?"), default=False) + has_time = models.BooleanField(_("Has time?"), default=False) + has_elevation = models.BooleanField(_("Has elevation?"), default=False) + time_regex = models.CharField(_("Time regex"), max_length=128, null=True, blank=True, choices=TIME_REGEX) + elevation_regex = models.CharField(_("Elevation regex"), max_length=128, null=True, blank=True) + + ptype = models.CharField(_("P-Type"), null=False, blank=False, max_length=255, default="gxp_wmscsource") + + default_style = models.ForeignKey( + Style, on_delete=models.SET_NULL, related_name="dataset_default_style", null=True, blank=True + ) + + styles = models.ManyToManyField(Style, related_name="dataset_styles") + + remote_service = models.ForeignKey("services.Service", null=True, blank=True, on_delete=models.CASCADE) + + charset = models.CharField(max_length=255, default="UTF-8") + + use_featureinfo_custom_template = models.BooleanField( + _("use featureinfo custom template?"), + help_text=_("specifies wether or not use a custom GetFeatureInfo template."), + default=False, + ) + featureinfo_custom_template = HTMLField( + _("featureinfo custom template"), + help_text=_("the custom GetFeatureInfo template HTML contents."), + unique=False, + blank=True, + null=True, + ) + + def is_vector(self): + return self.subtype in ["vector", "vector_time"] + + @property + def is_raster(self): + return self.subtype == "raster" + + @property + def display_type(self): + if self.subtype in ["vector", "vector_time"]: + return "Vector Data" + elif self.subtype == "raster": + return "Raster Data" + else: + return "Data" + + @property + def data_model(self): + if hasattr(self, "modeldescription_set"): + lmd = self.modeldescription_set.all() + if lmd.exists(): + return lmd.get().get_django_model() + + return None + + @property + def data_objects(self): + if self.data_model is not None: + return self.data_model.objects.using("datastore") + + return None + + @property + def attributes(self): + if self.attribute_set and self.attribute_set.count(): + _attrs = self.attribute_set + else: + _attrs = Attribute.objects.filter(dataset=self) + return _attrs.exclude(attribute="the_geom").order_by("display_order") + + @property + def service_typename(self): + return f"{self.remote_typename}:{self.alternate}" if self.remote_typename else self.alternate + + # layer geometry type. + @property + def gtype(self): + # return attribute type without 'gml:' and 'PropertyType' + if self.attribute_set and self.attribute_set.count(): + _attrs = self.attribute_set + else: + _attrs = Attribute.objects.filter(dataset=self) + if _attrs.filter(attribute="the_geom").exists(): + _att_type = _attrs.filter(attribute="the_geom").first().attribute_type + _gtype = re.match(r"gml:(.*)PropertyType", _att_type) + return _gtype.group(1) if _gtype else None + return None + + def get_base_file(self): + """Get the shp or geotiff file for this layer.""" + + # If there was no upload_session return None + try: + if self.upload_session is None: + return None, None + except Exception: + return None, None + + base_exts = [x.replace(".", "") for x in cov_exts + vec_exts] + base_files = self.upload_session.layerfile_set.filter(name__in=base_exts) + base_files_count = base_files.count() + + # If there are no files in the upload_session return None + if base_files_count == 0: + return None, None + + msg = f"There should only be one main file (.shp or .geotiff or .asc), found {base_files_count}" + assert base_files_count == 1, msg + + # we need to check, for shapefile, if column names are valid + list_col = None + if self.subtype in ["vector", "vector_time"]: + valid_shp, wrong_column_name, list_col = check_shp_columnnames(self) + if wrong_column_name: + msg = f"Shapefile has an invalid column name: {wrong_column_name}" + else: + msg = _("File cannot be opened, maybe check the encoding") + # AF: Removing assertion since if the original file does not exists anymore + # it won't be possible to update Metadata anymore + # assert valid_shp, msg + + # no error, let's return the base files + return base_files.get(), list_col + + def get_absolute_url(self): + return hookset.dataset_detail_url(self) + + @property + def capabilities_url(self): + from geonode.geoserver.helpers import get_dataset_capabilities_url + + return get_dataset_capabilities_url(self) + + @property + def dataset_ows_url(self): + from geonode.geoserver.helpers import get_layer_ows_url + + return get_layer_ows_url(self) + + @property + def embed_url(self): + try: + if self.service_typename: + return reverse("dataset_embed", kwargs={"layername": self.service_typename}) + except Exception as e: + logger.exception(e) + return None + + def attribute_config(self): + # Get custom attribute sort order and labels if any + cfg = {} + visible_attributes = self.attribute_set.visible() + if visible_attributes.exists(): + cfg["getFeatureInfo"] = { + "fields": [lyr.attribute for lyr in visible_attributes], + "propertyNames": {lyr.attribute: lyr.attribute_label for lyr in visible_attributes}, + "displayTypes": {lyr.attribute: lyr.featureinfo_type for lyr in visible_attributes}, + } + + if self.use_featureinfo_custom_template: + cfg["ftInfoTemplate"] = self.featureinfo_custom_template + + return cfg + + def __str__(self): + return str(self.alternate) + + class Meta(ResourceBase.Meta): + # custom permissions, + # change and delete are standard in django-guardian + permissions = ( + ("change_dataset_data", "Can edit layer data"), + ("change_dataset_style", "Can change layer style"), + ) + unique_together = ("store", "workspace", "name") + + # Permission Level Constants + # LEVEL_NONE inherited + LEVEL_READ = "dataset_readonly" + LEVEL_WRITE = "dataset_readwrite" + LEVEL_ADMIN = "dataset_admin" + + @property + def maps(self): + from geonode.maps.models import Map + + map_ids = list(self.maplayers.values_list("map__id", flat=True)) + return Map.objects.filter(id__in=map_ids) + + def get_linked_resources(self, as_target: bool = False): + ret = super().get_linked_resources(as_target) + + if as_target: + # create LinkedResources on the fly to report MapLayer relationship + res = (LinkedResource(source=map, target=self, internal=True) for map in self.maps) + ret = itertools.chain(ret, res) + + return ret + + @property + def download_url(self): + if self.subtype not in ["vector", "raster", "vector_time"]: + logger.info("Download URL is available only for datasets that have been harvested and copied locally") + return None + return build_absolute_uri(reverse("dataset_download", args=(self.alternate,))) + + @property + def maplayers(self): + from geonode.maps.models import MapLayer + + return MapLayer.objects.filter(name=self.alternate) + + @classproperty + def allowed_permissions(cls): + return { + "anonymous": VIEW_PERMISSIONS + DOWNLOAD_PERMISSIONS, + "default": OWNER_PERMISSIONS + DOWNLOAD_PERMISSIONS + DATASET_ADMIN_PERMISSIONS, + groups_settings.REGISTERED_MEMBERS_GROUP_NAME: OWNER_PERMISSIONS + + DOWNLOAD_PERMISSIONS + + DATASET_ADMIN_PERMISSIONS, + } + + @property + def class_name(self): + return self.__class__.__name__ + + def view_count_up(self, user, do_local=False): + """increase view counter, if user is not owner and not super + + @param user which views layer + @type User model + + @param do_local - do local counter update even if pubsub is enabled + @type bool + """ + if user == self.owner or user.is_superuser: + return + if not do_local: + from geonode.messaging import producer + + producer.viewing_dataset(str(user), str(self.owner), self.id) + + else: + Dataset.objects.filter(id=self.id).update(popular_count=models.F("popular_count") + 1) + + +class AttributeManager(models.Manager): + """Helper class to access filtered attributes""" + + def visible(self): + return self.get_queryset().filter(visible=True).order_by("display_order") + + +class Attribute(models.Model): + """ + Auxiliary model for storing layer attributes. + + This helps reduce the need for runtime lookups + to other servers, and lets users customize attribute titles, + sort order, and visibility. + """ + + dataset = models.ForeignKey( + Dataset, blank=False, null=False, unique=False, on_delete=models.CASCADE, related_name="attribute_set" + ) + attribute = models.CharField( + _("attribute name"), + help_text=_("name of attribute as stored in shapefile/spatial database"), + max_length=255, + blank=False, + null=True, + unique=False, + ) + description = models.CharField( + _("attribute description"), + help_text=_("description of attribute to be used in metadata"), + max_length=255, + blank=True, + null=True, + ) + attribute_label = models.CharField( + _("attribute label"), + help_text=_("title of attribute as displayed in GeoNode"), + max_length=255, + blank=True, + null=True, + unique=False, + ) + attribute_type = models.CharField( + _("attribute type"), + help_text=_("the data type of the attribute (integer, string, geometry, etc)"), + max_length=50, + blank=False, + null=False, + default="xsd:string", + unique=False, + ) + visible = models.BooleanField( + _("visible?"), help_text=_("specifies if the attribute should be displayed in identify results"), default=True + ) + display_order = models.IntegerField( + _("display order"), + help_text=_("specifies the order in which attribute should be displayed in identify results"), + default=1, + ) + + """ + Attribute FeatureInfo-Type list + """ + TYPE_PROPERTY = "type_property" + TYPE_HREF = "type_href" + TYPE_IMAGE = "type_image" + TYPE_VIDEO_MP4 = "type_video_mp4" + TYPE_VIDEO_OGG = "type_video_ogg" + TYPE_VIDEO_WEBM = "type_video_webm" + TYPE_VIDEO_3GP = "type_video_3gp" + TYPE_VIDEO_FLV = "type_video_flv" + TYPE_VIDEO_YOUTUBE = "type_video_youtube" + TYPE_AUDIO = "type_audio" + TYPE_IFRAME = "type_iframe" + + TYPES = ( + ( + TYPE_PROPERTY, + _("Label"), + ), + ( + TYPE_HREF, + _("URL"), + ), + ( + TYPE_IMAGE, + _( + "Image", + ), + ), + ( + TYPE_VIDEO_MP4, + _( + "Video (mp4)", + ), + ), + ( + TYPE_VIDEO_OGG, + _( + "Video (ogg)", + ), + ), + ( + TYPE_VIDEO_WEBM, + _( + "Video (webm)", + ), + ), + ( + TYPE_VIDEO_3GP, + _( + "Video (3gp)", + ), + ), + ( + TYPE_VIDEO_FLV, + _( + "Video (flv)", + ), + ), + ( + TYPE_VIDEO_YOUTUBE, + _( + "Video (YouTube/VIMEO - embedded)", + ), + ), + ( + TYPE_AUDIO, + _( + "Audio", + ), + ), + ( + TYPE_IFRAME, + _( + "IFRAME", + ), + ), + ) + featureinfo_type = models.CharField( + _("featureinfo type"), + help_text=_("specifies if the attribute should be rendered with an HTML widget on GetFeatureInfo template."), + max_length=255, + unique=False, + blank=False, + null=False, + default=TYPE_PROPERTY, + choices=TYPES, + ) + + # statistical derivations + count = models.IntegerField(_("count"), help_text=_("count value for this field"), default=1) + min = models.CharField( + _("min"), + help_text=_("minimum value for this field"), + max_length=255, + blank=False, + null=True, + unique=False, + default="NA", + ) + max = models.CharField( + _("max"), + help_text=_("maximum value for this field"), + max_length=255, + blank=False, + null=True, + unique=False, + default="NA", + ) + average = models.CharField( + _("average"), + help_text=_("average value for this field"), + max_length=255, + blank=False, + null=True, + unique=False, + default="NA", + ) + median = models.CharField( + _("median"), + help_text=_("median value for this field"), + max_length=255, + blank=False, + null=True, + unique=False, + default="NA", + ) + stddev = models.CharField( + _("standard deviation"), + help_text=_("standard deviation for this field"), + max_length=255, + blank=False, + null=True, + unique=False, + default="NA", + ) + sum = models.CharField( + _("sum"), + help_text=_("sum value for this field"), + max_length=255, + blank=False, + null=True, + unique=False, + default="NA", + ) + unique_values = models.TextField(_("unique values for this field"), null=True, blank=True, default="NA") + last_stats_updated = models.DateTimeField( + _("last modified"), default=now, help_text=_("date when attribute statistics were last updated") + ) # passing the method itself, not + + objects = AttributeManager() + + def __str__(self): + return str(self.attribute_label if self.attribute_label else self.attribute) + + def unique_values_as_list(self): + return self.unique_values.split(",") diff --git a/geonode/layers/populate_datasets_data.py b/geonode/layers/populate_datasets_data.py new file mode 100644 index 0000000..51fc241 --- /dev/null +++ b/geonode/layers/populate_datasets_data.py @@ -0,0 +1,137 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.conf import settings +from django.contrib.auth import get_user_model + +from geonode.layers.models import Style, Attribute, Dataset + +ogc_location = settings.OGC_SERVER["default"]["LOCATION"] + + +styles = [ + { + "name": "test_style_1", + "sld_url": f"{ogc_location}rest/styles/test_style.sld", + "sld_body": 'test_style_1\ + test_style_1name\ + #888800\ + #ffffbb\ + 0.7\ + \ + ', + }, + { + "name": "test_style_2", + "sld_url": f"{ogc_location}rest/styles/test_style.sld", + "sld_body": 'test_style_2\ + test_style_2name\ + #888800\ + #ffffbb\ + 0.7\ + ', + }, + { + "name": "test_style_3", + "sld_url": f"{ogc_location}rest/styles/test_style.sld", + "sld_body": 'test_style_3\ + test_style_3name\ + #888800\ + #ffffbb0.7\ + ', + }, + { + "name": "Evaluación", + "sld_url": f"{ogc_location}rest/styles/test_style.sld", + "sld_body": '\ + test_style_3test_style_3\ + name\ + #888800#ffffbb0.7\ + \ + ', + }, +] + +attributes = [ + { + "attribute": "N\xfamero_De_M\xe9dicos", + "attribute_label": "N\xfamero_De_M\xe9dicos", + "attribute_type": "xsd:string", + "visible": True, + "display_order": 4, + }, + { + "attribute": "the_geom", + "attribute_label": "Shape", + "attribute_type": "gml:Geometry", + "visible": False, + "display_order": 3, + }, + { + "attribute": "description", + "attribute_label": "Description", + "attribute_type": "xsd:string", + "visible": True, + "display_order": 2, + }, + { + "attribute": "place_name", + "attribute_label": "Place Name", + "attribute_type": "xsd:string", + "visible": True, + "display_order": 1, + }, +] + + +def create_dataset_data(object_id=None, owner=None): + layer = Dataset.objects.get(pk=object_id) if object_id else Dataset.objects.all().first() + if not owner: + owner = get_user_model().objects.get(username="admin") + for style in styles: + new_style, created = Style.objects.get_or_create( + name=style["name"], defaults=dict(sld_url=style["sld_url"], sld_body=style["sld_body"]) + ) + if new_style not in layer.styles.all(): + layer.styles.add(new_style) + layer.default_style = new_style + layer.owner = owner + layer.save() + + for attr in attributes: + Attribute.objects.update_or_create( + dataset=layer, + attribute=attr["attribute"], + attribute_label=attr["attribute_label"], + attribute_type=attr["attribute_type"], + visible=attr["visible"], + display_order=attr["display_order"], + ) diff --git a/geonode/layers/tasks.py b/geonode/layers/tasks.py new file mode 100644 index 0000000..66a131d --- /dev/null +++ b/geonode/layers/tasks.py @@ -0,0 +1,53 @@ +######################################################################### +# +# Copyright (C) 2017 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""celery tasks for geonode.layers.""" +from geonode.celery_app import app +from celery.utils.log import get_task_logger + +from geonode.layers.models import Dataset +from geonode.resource.manager import resource_manager + +logger = get_task_logger(__name__) + + +@app.task( + bind=True, + name="geonode.layers.tasks.delete_dataset", + queue="cleanup", + expires=30, + time_limit=600, + acks_late=False, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 5}, + retry_backoff=3, + retry_backoff_max=30, + retry_jitter=False, +) +def delete_dataset(self, dataset_id): + """ + Deletes a layer. + """ + try: + layer = Dataset.objects.get(id=dataset_id) + except Dataset.DoesNotExist: + logger.warning(f"Layers {dataset_id} does not exist!") + return + logger.debug(f"Deleting Dataset {layer}") + resource_manager.delete(uuid=layer.uuid, instance=layer) diff --git a/geonode/layers/templates/datasets/dataset_base.html b/geonode/layers/templates/datasets/dataset_base.html new file mode 100644 index 0000000..df46dd7 --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_base.html @@ -0,0 +1,11 @@ +{% extends "geonode_base.html" %} +{% load i18n %} + +{% block title %} {{ block.super }} {% endblock %} + +{% block body_class %}data{% endblock body_class %} + +{% block body_outer %} + {% block body %}{% endblock body %} + {% block sidebar %}{% endblock sidebar %} +{% endblock body_outer %} diff --git a/geonode/layers/templates/datasets/dataset_embed.html b/geonode/layers/templates/datasets/dataset_embed.html new file mode 100644 index 0000000..d2db1ee --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_embed.html @@ -0,0 +1,3 @@ +{% load i18n %} +{% load client_lib_tags %} +{% get_dataset_embed %} diff --git a/geonode/layers/templates/datasets/dataset_granule_remove.html b/geonode/layers/templates/datasets/dataset_granule_remove.html new file mode 100644 index 0000000..6eb2efa --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_granule_remove.html @@ -0,0 +1,41 @@ +{% extends "datasets/dataset_base.html" %} +{% load i18n %} +{% load client_lib_tags %} + +{% block title %}{{ dataset.alternate }} — {{ block.super }}{% endblock %} + +{% block body %} +
      + {% trans "Explore Datasets" %} +

      {% trans "Remove Mosaic Granules" %}

      +
      +
      +
      +

      + {% blocktrans with dataset.alternate as dataset_title %} + Are you sure you want to remove Granule {{ granule_id }} of the Mosaic {{ dataset_title }}? + {% endblocktrans %} +

      +
      + {% csrf_token %} + +
      + + {% trans "No, don't remove it" %} +
      +
      +
      + {% if dataset.maplayers %} +
      +
        +
      • {% trans "This action affects the following maps:" %}
        • + {% for map_layer in dataset.maplayers %} +
      • {{ map_layer.map.title }}
        • + {% empty %} +
      • {% trans "No maps are using this dataset" %}
        • + {% endfor %} +
      +
      + {% endif %} +
      +{% endblock %} diff --git a/geonode/layers/templates/datasets/dataset_metadata.html b/geonode/layers/templates/datasets/dataset_metadata.html new file mode 100644 index 0000000..69fe111 --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_metadata.html @@ -0,0 +1,111 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load bootstrap_tags %} +{% load base_tags %} +{% load guardian_tags %} +{% load floppyforms %} +{% load client_lib_tags %} + +{% block title %}{{ dataset.alternate }} — {{ block.super }}{% endblock title %} + +{% block head %} + {% include "ol/datasets/dataset_ol2_map.html" %} +{{ block.super }} +{% endblock head %} + +{% block body_class %}data{% endblock body_class %} + +{% block body_outer %} +
      +

      {% trans "Metadata" %} {% blocktrans with dataset.title as dataset_title %}for {{ dataset_title }}{% endblocktrans %}

      +
      + {% trans "Completeness" %} +
      + + +
      +
      +
      + 70% +
      +
      +
      + +
      + +
      + {% if dataset.metadata_uploaded_preserve %} +

      {% blocktrans %}Note: this layer's orginal metadata was populated and preserved by importing a metadata XML file. + This metadata cannot be edited.{% endblocktrans %}

      + {% elif dataset.metadata_uploaded %} +

      {% blocktrans %}Note: this layer's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}

      + {% endif %} + + {% if dataset_form.errors or attribute_form.errors or category_form.errors or metadata_author_form.errors or poc.errors or tkeywords_form.errors %} +

      {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %}

      +
        + {% if metadata_author_form.errors %} +
      • {% trans "Metadata Author" %}
        • + {{ metadata_author_form.errors }} + {% endif %} + {% if poc_form.errors %} +
      • {% trans "Point of Contact" %}
        • + {{ poc_form.errors }} + {% endif %} + {% for field in dataset_form %} + {% if field.errors %} +
      • {{ field.label }}
        • + {% endif %} + {% endfor %} + {% if not attribute_form.is_valid %} +
      • {% trans "Attributes" %}
        • + {% for field in attribute_form %} + {% if field.errors %} +
      • {{ field.errors }}
        • + {% endif %} + {% endfor %} + {% endif %} + {% if category_form.errors %} +
      • {{ category_form.errors.as_ul }}
        • + {% endif %} + {% if tkeywords_form.errors %} +
      • {{ tkeywords_form.errors.as_ul }}
        • + {% endif %} +
      + {% endif %} + + {% csrf_token %} +
      + {% form dataset_form using panel_template %} + {# dataset_form|as_bootstrap #} +
      + +
      +
      + + + + {% if not dataset.metadata_uploaded_preserve %} +
      + {% trans "Return to Dataset" %} + + + >" %}"/> +
      + {% endif %} +
      +
      +
      + +{{ block.super }} +{% endblock body_outer %} diff --git a/geonode/layers/templates/datasets/dataset_metadata_advanced.html b/geonode/layers/templates/datasets/dataset_metadata_advanced.html new file mode 100644 index 0000000..48379c9 --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_metadata_advanced.html @@ -0,0 +1,257 @@ +{% extends "metadata_base.html" %} +{% load i18n %} +{% load static %} +{% load base_tags %} +{% load bootstrap_tags %} +{% load guardian_tags %} +{% load client_lib_tags %} + +{% block title %}{{ dataset.title }} — {{ block.super }}{% endblock %} + +{% block body_class %}data{% endblock %} + +{% block body_outer %} + +{{ block.super }} + + + + + + + + + + +
      + {% trans "Explore Datasets" %} +

      {% trans "Edit Metadata" %}

      +
      +
      +
      +

      + {% blocktrans with dataset.title as dataset_title %} + Editing details for {{ dataset_title }} + {% endblocktrans %} +

      + {% block advanced_edit_form %} +
      + + {% block metadata_uploaded_check %} + {% if dataset.metadata_uploaded_preserve %} +

      {% blocktrans %}Note: this layer's orginal metadata was populated and preserved by importing a metadata XML file. + This metadata cannot be edited.{% endblocktrans %}

      + {% elif dataset.metadata_uploaded %} +

      {% blocktrans %}Note: this layer's orginal metadata was populated by importing a metadata XML file. + GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin Core metadata elements. + Some of your original metadata may have been lost.{% endblocktrans %}

      + {% endif %} + {% endblock metadata_uploaded_check %} + + {% block dataset_form_errors %} + {% if dataset_form.errors or attribute_form.errors or category_form.errors or metadata_author_form.errors or poc.errors %} +

      {% blocktrans %}Error updating metadata. Please check the following fields: {% endblocktrans %}

      +
        + {% if metadata_author_form.errors %} +
      • {% trans "Metadata Author" %}
        • + {{ metadata_author_form.errors }} + {% endif %} + {% if poc_form.errors %} +
      • {% trans "Point of Contact" %}
        • + {{ poc_form.errors }} + {% endif %} + {% for field in dataset_form %} + {% if field.errors %} +
      • {{ field.label }}
        • + {% endif %} + {% endfor %} + {% if not attribute_form.is_valid %} +
      • {% trans "Attributes" %}
        • + {% for field in attribute_form %} + {% if field.errors %} +
      • {{ field.errors }}
        • + {% endif %} + {% endfor %} + {% endif %} + {% if category_form.errors %} +
      • {{ category_form.errors.as_ul }}
        • + {% endif %} +
      + {% endif %} + {% endblock dataset_form_errors %} + + {% if not dataset.metadata_uploaded_preserve %} +
      + {% trans "Return to Dataset" %} + +
      + {% endif %} + + {% csrf_token %} + +
      + {% block dataset_fields %} + {% for field in dataset_form %} + {% if field.name != 'use_featureinfo_custom_template' and field.name != 'featureinfo_custom_template' and field.name not in ADVANCED_EDIT_EXCLUDE_FIELD %} + {% if field.name == 'featured' and not user.is_superuser %} + {% else %} +
      +
      + + {{ field }} +
      +
      + {% endif %} + {% endif %} + {% endfor %} + {% endblock dataset_fields %} + + + {% block thesauri %} + {# dataset_form|as_bootstrap #} + {% if THESAURI_FILTERS %} + {% for field in tkeywords_form %} +
      +

      + + {{ field }} +

      +
      + {% endfor %} + {% endif %} + {% endblock thesauri %} +
      + + {% block category %} +
      +
      + +
      + {% autoescape off %} + {% for choice in category_form.category_choice_field.field.choices %} +
      + +
      + {% endfor %} + {% endautoescape %} +
      +
      + {% endblock category %} + + {% block other_tab %}{% endblock other_tab %} + + {% block attributes %} +
      +
      {% trans "Attributes" %}
      + + +
      + {{ attribute_form.management_form }} + + + + + + + + + + {% for form in attribute_form.forms %} + {% if form.attribute %} + + + + + + + + + {% endif %} + {% endfor %} +
      {% trans "Attribute" %}{% trans "Label" %}{% trans "Description" %}{% trans "Display Order" %}{% trans "Display Type" %}{% trans "Visible" %}
      {{form.id}}
      {{form.attribute}}      		{{form.attribute_label}}{{form.description}}{{form.display_order}}{{form.featureinfo_type}}{{form.visible}}
      +
      + {% endblock attributes %} + + {% block dataset_custom_template %} +
      + {{dataset_form.featureinfo_custom_template}} +
      + {% endblock dataset_custom_template %} + + {% block point_of_contact %} + + {% endblock point_of_contact %} + {% block metadata_provider %} + + {% endblock metadata_provider %} + + {% if not dataset.metadata_uploaded_preserve %} + {% block form_actions %} +
      + {% trans "Return to Dataset" %} + +
      + {% endblock form_actions %} + {% endif %} +
      +
      +
      + {% endblock advanced_edit_form %} +
      +
      + + + +{% endblock %} + +{% block extra_script %} + {{ block.super }} + +{% endblock %} \ No newline at end of file diff --git a/geonode/layers/templates/datasets/dataset_metadata_detail.html b/geonode/layers/templates/datasets/dataset_metadata_detail.html new file mode 100644 index 0000000..340824f --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_metadata_detail.html @@ -0,0 +1,6 @@ +{% extends "metadata_detail.html" %} +{% load i18n %} +{% block metaget_absolute_url %} +
      {% trans "Metadata Page" %}
      +
      {% url "dataset_metadata_detail" resource.alternate %}
      +{% endblock metaget_absolute_url %} \ No newline at end of file diff --git a/geonode/layers/templates/datasets/dataset_metadata_upload.html b/geonode/layers/templates/datasets/dataset_metadata_upload.html new file mode 100644 index 0000000..d2d832e --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_metadata_upload.html @@ -0,0 +1,90 @@ +{% extends "upload/dataset_upload_metadata_base.html" %} +{% load i18n %} +{% load static %} +{% load bootstrap_tags %} +{% load pagination_tags %} +{% load base_tags %} +{% load guardian_tags %} +{% load client_lib_tags %} + +{% block title %} {% trans "Upload Dataset Metadata" %} - {{ block.super }} {% endblock %} + +{% block body_class %}layers upload{% endblock %} + + +{% block head %} + +{{ block.super }} +{% endblock %} + +{% block body_outer %} + +
      + {% trans "Explore Datasets" %} + {% trans "Return to Dataset" %} +

      {% trans "Upload Dataset Metadata" %} {% trans "(XML - ISO, FGDC, ebRIM, Dublin Core)" %}

      +
      + +
      +
      + {% block additional_info %}{% endblock %} + + {% if errors %} +
      + {% for error in errors %} +

      {{ error }}

      + {% endfor %} +
      + {% endif %} + +
      + +
      +


      {% trans "Drop files here" %}

      +
      + +

      {% trans " or select them one by one:" %}

      + +
      + + + +
      + +
      +
        +

        {% trans "Files to be uploaded" %}

        +
        + +
        + +
        + {% trans "Clear" %} + {% trans "Upload files" %} +
        +
        +
        +{% endblock %} + +{% block extra_script %} + {{ block.super }} + + +{% endblock extra_script %} diff --git a/geonode/layers/templates/datasets/dataset_style_upload.html b/geonode/layers/templates/datasets/dataset_style_upload.html new file mode 100644 index 0000000..86d8a8f --- /dev/null +++ b/geonode/layers/templates/datasets/dataset_style_upload.html @@ -0,0 +1,91 @@ +{% extends "upload/dataset_upload_metadata_base.html" %} +{% load i18n %} +{% load static %} +{% load bootstrap_tags %} +{% load pagination_tags %} +{% load base_tags %} +{% load guardian_tags %} +{% load client_lib_tags %} + +{% block title %} {% trans "Upload Dataset Metadata" %} - {{ block.super }} {% endblock %} + +{% block body_class %}layers upload{% endblock %} + + +{% block head %} + +{{ block.super }} +{% endblock %} + +{% block body_outer %} + +
        + {% trans "Explore Datasets" %} + {% trans "Return to Dataset" %} +

        {% trans "Upload Dataset Style" %} {% trans "(SLD - Style Layer Descriptor 1.0, 1.1)" %}

        +
        + +
        +
        + {% block additional_info %}{% endblock %} + + {% if errors %} +
        + {% for error in errors %} +

        {{ error }}

        + {% endfor %} +
        + {% endif %} + +
        + +
        +


        {% trans "Drop files here" %}

        +
        + +

        {% trans " or select them one by one:" %}

        + +
        + + + +
        + +
        +
          +

          {% trans "Files to be uploaded" %}

          +
          + +
          + +
          +
          {% trans "WARNING" %}: {% trans "This will most probably overwrite the current default style!" %}
          + {% trans "Clear" %} + {% trans "Upload files" %} +
          +
          +
          +{% endblock %} + +{% block extra_script %} + {{ block.super }} + + +{% endblock extra_script %} diff --git a/geonode/layers/templates/datasets/wps_execute_gs_aggregate.xml b/geonode/layers/templates/datasets/wps_execute_gs_aggregate.xml new file mode 100644 index 0000000..4110e9d --- /dev/null +++ b/geonode/layers/templates/datasets/wps_execute_gs_aggregate.xml @@ -0,0 +1,84 @@ + + + gs:Aggregate + + + features + + + + + + + + + + aggregationAttribute + + {{ field }} + + + + function + + Count + + + + function + + Min + + + + function + + Max + + + + function + + Average + + + + function + + Median + + + + function + + StdDev + + + + function + + Sum + + + + singlePass + + True + + + + + + + result + + + diff --git a/geonode/layers/templates/datasets/wps_execute_gs_unique.xml b/geonode/layers/templates/datasets/wps_execute_gs_unique.xml new file mode 100644 index 0000000..ec4a5ef --- /dev/null +++ b/geonode/layers/templates/datasets/wps_execute_gs_unique.xml @@ -0,0 +1,27 @@ + + + gs:Unique + + + features + + + + + + + + + + attribute + + {{ field }} + + + + + + result + + + diff --git a/geonode/layers/templates/layouts/panels.html b/geonode/layers/templates/layouts/panels.html new file mode 100644 index 0000000..58066f2 --- /dev/null +++ b/geonode/layers/templates/layouts/panels.html @@ -0,0 +1,828 @@ +{% load i18n %} +{% load static %} +{% load floppyforms %} +{% load contact_roles %} + + + + + + + + + + + + + + + + + + + +{% block body_outer %} + + +
          +
          +{% block edit_progress %} + +
          + +
          + {% trans "Mandatory" %} +
          +
          + {% trans "Mandatory" %} +
          + {% if UI_REQUIRED_FIELDS %} +
          + {% trans "Mandatory" %} +
          +
          + {% trans "Optional" %} +
          + {% else %} +
          + {% trans "Optional" %} +
          + {% endif %} + +
          +{% endblock edit_progress %} + {% block breadcrumbs %} + + {% endblock breadcrumbs %} + {% block mandatory %} +
          + +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + {% block thumbnail %} +
          + +
          +
          +
          +
          + +
          + +
          +
          + {% endblock thumbnail %} +
          + {% block dataset_title %} +
          + + + {{ dataset_form.title }} +
          + {% endblock dataset_title %} + {% block dataset_abstract %} +
          + + + {{ dataset_form.abstract }} +
          + {% endblock dataset_abstract %} +
          +
          + {% block dataset_date_type %} +
          + + + {{ dataset_form.date_type }} +
          + {% endblock dataset_date_type %} + {% block dataset_date %} +
          + + + {{ dataset_form.date }} +
          + {% endblock dataset_date %} + {% block dataset_category %} +
          + + +
          + {% endblock dataset_category %} + {% block dataset_group %} +
          + + +
          + {% endblock dataset_group %} + {% block dataset_free_keyword %} +
          + + {{ dataset_form.keywords }} +
          + {% endblock dataset_free_keyword %} + {% if THESAURI_FILTERS %} +
          + {{tkeywords_form.as_p}} +
          + {% endif %} +
          +
          +
          +
          +
          +
          +
          +
          + {% endblock %} + {% block advanced %} +
          +
          +
          + +
          +
          +
          + {% block dataset_attributes %} +
          +
          + + + {{ dataset_form.language }} +
          +
          + + + {{ dataset_form.license }} +
          +
          + + {{ dataset_form.attribution }} +
          +
          + {% endblock dataset_attributes %} + {% block dataset_regions %} +
          +
          + + {{ dataset_form.regions }} +
          +
          + + + {{ dataset_form.data_quality_statement }} +
          +
          + {% endblock dataset_regions %} + {% block dataset_constraints %} +
          +
          + + + {{ dataset_form.restriction_code_type }} +
          +
          + + + {{ dataset_form.constraints_other }} +
          +
          + {% endblock dataset_constraints %} +
          +
          +
          +
          +
          +
          +
          +
          + {% endblock %} + {% block other_tab %}{% endblock other_tab %} + {% block ownership %} +
          +
          +
          +
          +

          {% trans "Other, Optional, Metadata" %}

          + {% block dataset_edition %} +
          + + + {{ dataset_form.edition }} +
          + {% endblock dataset_edition %} + {% block dataset_doi %} +
          + + {{ dataset_form.doi }} +
          + {% endblock dataset_doi %} + {% block dataset_purpose %} +
          + + + {{ dataset_form.purpose }} +
          + {% endblock dataset_purpose %} + {% block dataset_supplemental_information %} +
          + + + {{ dataset_form.supplemental_information }} +
          + {% endblock dataset_supplemental_information %} +
          +
          + {% block dataset_temporal_extent_start %} +
          +
          + + + {{ dataset_form.temporal_extent_start }} +
          +
          + {% endblock dataset_temporal_extent_start %} + {% block dataset_temporal_extent_end %} +
          +
          + + + {{ dataset_form.temporal_extent_end }} +
          +
          + {% endblock dataset_temporal_extent_end %} +
          + {% block dataset_maintenance_frequency %} +
          + + + {{ dataset_form.maintenance_frequency }} +
          + {% endblock dataset_maintenance_frequency %} + {% block dataset_spatial_representation_type %} +
          + + + {{ dataset_form.spatial_representation_type }} +
          + {% endblock dataset_spatial_representation_type %} + {% block layer_extra_metadata %} +
          + + {{ dataset_form.extra_metadata }} +
          + {% endblock layer_extra_metadata %} + {% block dataset_linked_resources %} +
          + + {{ dataset_form.linked_resources }} +
          + {% endblock dataset_linked_resources %} +
          +
          + +
          +
          +
          {% trans "Responsible Parties" %}
          + {% block dataset_poc %} +
          + + {{ dataset_form.poc }} +
          + {% endblock dataset_poc %} +
          +
          +
          {% trans "Responsible and Permissions" %}
          +
          + {% block dataset_owner %} +
          + + {{ dataset_form.owner }} +
          + {% endblock dataset_owner %} +
          +
          + {% trans "toggle more Contact Roles" %} + {% block dataset_more_contact_roles %} +
          +
          {% trans "more metadata contact roles" %}
          + {% for contact_role in UI_ROLES_IN_TOGGLE_VIEW %} + {% getattribute dataset_form contact_role as cr %} +
          +
          + + {{ cr}} +
          +
          + {% endfor %} +
          +
          + {% endblock dataset_more_contact_roles %} +
          + +
          +
          +
          +
          + {% endblock %} + {% block dataset %} +
          +
          +
          +
          +
          + + +
          + {{ attribute_form.management_form }} + + + + + + + + + + + {% for form in attribute_form.forms %} + {% if form.attribute %} + + + + + + + + + + {% endif %} + {% endfor %} +
          {% trans "Attribute" %}{% trans "Label" %}{% trans "Description" %}{% trans "Display Order" %}{% trans "Display Type" %}{% trans "Visible" %}
          {{form.id}}
          {{form.attribute}}          		{{form.attribute_label}}{{form.description}}{{form.display_order}}{{form.featureinfo_type}}{{form.visible}}
          +
          +
          + {{dataset_form.featureinfo_custom_template}} +
          +
          +
          +
          +
          +
          +
          + {% endblock %} +
          + {% block extra_metadata_content %} + {% endblock %} +
          +
          + + {% if metadataxsl %} + + {% else %} + + {% endif %} +
          +
          + +
          +
          +
          +
          +
          +
          {% trans "Publishing" %}
          +
          +
          + + {{ dataset_form.metadata_uploaded_preserve }} +
          +
          + + {{ dataset_form.is_approved }} +
          +
          + + {{ dataset_form.is_published }} +
          + {% if user.is_superuser %} +
          + + {{ dataset_form.featured }} +
          + {% endif %} +
          + + {{ dataset_form.advertised }} +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          {% trans "Other Settings" %}
          +
          +
          + + {{ dataset_form.is_mosaic }} +
          +
          + + {{ dataset_form.has_time }} +
          +
          + + {{ dataset_form.has_elevation }} +
          + +
          + + {{ dataset_form.time_regex }} +
          +
          + + {{ dataset_form.elevation_regex }} +
          +
          +
          +
          +
          +
          +
          + {% if resource.is_vector %} +
          +
          +
          +
          +
          +
          {% trans "Time series settings" %}
          +
          +
          + + {{timeseries_form.attribute}} +
          +
          + + {{timeseries_form.end_attribute}} +
          + + {{timeseries_form.presentation}} +
          +
          + {{timeseries_form.precision_value}}
          + + {{timeseries_form.precision_step}} +
          +
          +
          +
          +
          +
          +
          +
          + {% endif %} +
          + + + +{% endblock %} + + diff --git a/geonode/layers/templatetags/__init__.py b/geonode/layers/templatetags/__init__.py new file mode 100644 index 0000000..79177e0 --- /dev/null +++ b/geonode/layers/templatetags/__init__.py @@ -0,0 +1,18 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### diff --git a/geonode/layers/templatetags/contact_roles.py b/geonode/layers/templatetags/contact_roles.py new file mode 100644 index 0000000..240d63d --- /dev/null +++ b/geonode/layers/templatetags/contact_roles.py @@ -0,0 +1,43 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django import template + +register = template.Library() + + +@register.filter(is_safe=True) +def get_contact_role_id(form, contact): + return id(form.fields[contact]) + + +@register.filter(is_safe=True) +def get_contact_role_label(form, contact): + return form.fields[contact].label + + +@register.filter(is_safe=True) +def get_contact_role_value(form, contact): + return form[contact] + + +@register.simple_tag +def getattribute(form, contact): + """Gets an attribute of an object dynamically from a string name""" + return form[contact] diff --git a/geonode/layers/templatetags/dataset_tags.py b/geonode/layers/templatetags/dataset_tags.py new file mode 100644 index 0000000..fc18af2 --- /dev/null +++ b/geonode/layers/templatetags/dataset_tags.py @@ -0,0 +1,83 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import logging + +from django import template +from django.conf import settings + +from urllib.parse import urlparse, parse_qs +from inflection import camelize as do_camelize + +logger = logging.getLogger(__name__) +register = template.Library() + + +@register.filter(is_safe=True) +def lower(value): # Only one argument. + """Converts a string into all lowercase""" + return value.lower() + + +@register.filter(is_safe=True) +def camelize(value): # Only one argument. + if str(value)[0].isdigit(): + return value + else: + """Converts a string into all camel""" + return do_camelize(value) + + +@register.filter(is_safe=True) +def crs_labels(value): # Only one argument. + """Converts a EPSG SRS ID into a human readable string""" + if value in settings.EPSG_CODE_MATCHES.keys(): + return settings.EPSG_CODE_MATCHES[value] + return value + + +@register.simple_tag +def get_all_resource_styles(resource): + if resource and resource.default_style: + return set( + list(resource.styles.all()) + + [ + resource.default_style, + ] + ) + else: + return set() + + +@register.simple_tag +def get_sld_name_from_url(sld_url): + if sld_url: + return os.path.basename(urlparse(sld_url).path).split(".")[0] + else: + return sld_url + + +@register.simple_tag +def get_style_name_from_legend_url(legend_url): + try: + parsed = urlparse(legend_url) + return parse_qs(parsed.query)["STYLE"][0] + except Exception as e: + logger.error(e) + return None diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py new file mode 100644 index 0000000..d11ad69 --- /dev/null +++ b/geonode/layers/tests.py @@ -0,0 +1,1879 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import itertools +import os +import shutil +import logging + +from uuid import uuid4 +from unittest.mock import MagicMock, patch +from collections import namedtuple + +from django.urls import reverse +from django.test import TestCase +from django.forms import ValidationError +from django.test.client import RequestFactory +from django.core.management import call_command +from django.contrib.auth.models import Group +from django.contrib.gis.geos import Polygon +from django.db.models import Count +from django.contrib.auth import get_user_model +from django.http import HttpResponse + +from django.conf import settings +from django.test.utils import override_settings +from django.contrib.admin.sites import AdminSite +from geonode.geoserver.createlayer.utils import create_dataset + +from geonode.layers import utils +from geonode.layers.utils import clear_dataset_download_handlers +from geonode.base import enumerations +from geonode.layers.apps import DatasetAppConfig +from geonode.layers.admin import DatasetAdmin +from geonode.decorators import on_ogc_backend +from geonode.maps.models import Map, MapLayer +from geonode.utils import DisableDjangoSignals, mkdtemp +from geonode.layers.views import _resolve_dataset +from geonode import GeoNodeException, geoserver +from geonode.people.utils import get_valid_user +from geonode.people import Roles +from guardian.shortcuts import get_anonymous_user +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.resource.manager import resource_manager +from geonode.tests.utils import NotificationsTestsHelper +from geonode.layers.models import Dataset, Style, Attribute +from geonode.layers.forms import DatasetForm, DatasetTimeSerieForm, JSONField +from geonode.layers.populate_datasets_data import create_dataset_data +from geonode.base.models import TopicCategory, License, Region, Link +from geonode.utils import check_ogc_backend, set_resource_default_links +from geonode.layers.metadata import convert_keyword, set_metadata, parse_metadata +from geonode.groups.models import GroupProfile + +from geonode.layers.utils import ( + dataset_type, + get_files, + get_valid_name, + get_valid_dataset_name, + surrogate_escape_string, +) + +from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset +from geonode.layers.download_handler import DatasetDownloadHandler + +logger = logging.getLogger(__name__) + + +class DatasetsTest(GeoNodeBaseTestSupport): + """Tests geonode.layers app/module""" + + type = "dataset" + + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.user = "admin" + self.passwd = "admin" + self.anonymous_user = get_anonymous_user() + self.exml_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" + self.sld_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_sld.sld" + self.maxDiff = None + self.sut = create_single_dataset("single_point") + create_dataset_data(self.sut.resourcebase_ptr_id) + create_dataset_data(Dataset.objects.first().resourcebase_ptr_id) + self.r = namedtuple("GSCatalogRes", ["resource"]) + + site = AdminSite() + self.admin = DatasetAdmin(Dataset, site) + + self.request_admin = RequestFactory().get("/admin") + self.request_admin.user = get_user_model().objects.get(username="admin") + + # Admin Tests + + def test_admin_save_model(self): + obj = Dataset.objects.first() + self.assertEqual(len(obj.keywords.all()), 2) + form = self.admin.get_form(self.request_admin, obj=obj, change=True) + self.admin.save_model(self.request_admin, obj, form, True) + + def test_default_sourcetype(self): + obj = Dataset.objects.first() + self.assertEqual(obj.sourcetype, enumerations.SOURCE_TYPE_LOCAL) + + # Data Tests + + def test_describe_data_2(self): + """/data/geonode:CA/metadata -> Test accessing the description of a layer""" + self.assertEqual(10, get_user_model().objects.all().count()) + response = self.client.get(reverse("dataset_metadata", args=("geonode:CA",))) + # Since we are not authenticated, we should not be able to access it + self.assertEqual(response.status_code, 302) + # but if we log in ... + self.client.login(username="admin", password="admin") + # ... all should be good + response = self.client.get(reverse("dataset_metadata", args=("geonode:CA",))) + self.assertEqual(response.status_code, 200) + + def test_describe_data_3(self): + """/data/geonode:CA/metadata_detail -> Test accessing the description of a layer""" + self.client.login(username="admin", password="admin") + # ... all should be good + response = self.client.get(reverse("dataset_metadata_detail", args=("geonode:CA",))) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Approved", count=1, status_code=200, msg_prefix="", html=False) + self.assertContains(response, "Published", count=1, status_code=200, msg_prefix="", html=False) + self.assertContains(response, "Featured", count=1, status_code=200, msg_prefix="", html=False) + self.assertContains(response, "
          Group
          ", count=0, status_code=200, msg_prefix="", html=False) + + # ... now assigning a Group to the Dataset + lyr = Dataset.objects.get(alternate="geonode:CA") + group = Group.objects.first() + lyr.group = group + lyr.save() + response = self.client.get(reverse("dataset_metadata_detail", args=("geonode:CA",))) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "
          Group
          ", count=1, status_code=200, msg_prefix="", html=False) + lyr.group = None + lyr.save() + + # Dataset Tests + + def test_dataset_name_clash(self): + _ll_1 = Dataset.objects.create( + uuid=str(uuid4()), + owner=get_user_model().objects.get(username=self.user), + name="states", + store="geonode_data", + subtype="vector", + alternate="geonode:states", + ) + _ll_2 = Dataset.objects.create( + uuid=str(uuid4()), + owner=get_user_model().objects.get(username=self.user), + name="geonode:states", + store="httpfooremoteservce", + subtype="remote", + alternate="geonode:states", + ) + _ll_1.set_permissions({"users": {"bobby": ["base.view_resourcebase"]}}) + _ll_2.set_permissions({"users": {"bobby": ["base.view_resourcebase"]}}) + self.client.login(username="bobby", password="bob") + _request = self.client.request() + _request.user = get_user_model().objects.get(username="bobby") + _ll = _resolve_dataset(_request, alternate="geonode:states") + self.assertIsNotNone(_ll) + self.assertEqual(_ll.name, _ll_1.name) + + def test_describe_data(self): + """/data/geonode:CA/metadata -> Test accessing the description of a layer""" + self.assertEqual(10, get_user_model().objects.all().count()) + response = self.client.get(reverse("dataset_metadata", args=("geonode:CA",))) + # Since we are not authenticated, we should not be able to access it + self.assertEqual(response.status_code, 302) + # but if we log in ... + self.client.login(username="admin", password="admin") + # ... all should be good + response = self.client.get(reverse("dataset_metadata", args=("geonode:CA",))) + self.assertEqual(response.status_code, 200) + + def test_dataset_attributes(self): + lyr = Dataset.objects.all().first() + # There should be a total of 3 attributes + self.assertEqual(len(lyr.attribute_set.all()), 4) + # 2 out of 3 attributes should be visible + custom_attributes = lyr.attribute_set.visible() + self.assertEqual(len(custom_attributes), 3) + # place_ name should come before description + self.assertEqual(custom_attributes[0].attribute_label, "Place Name") + self.assertEqual(custom_attributes[1].attribute_label, "Description") + self.assertEqual(custom_attributes[2].attribute, "N\xfamero_De_M\xe9dicos") + # TODO: do test against layer with actual attribute statistics + self.assertEqual(custom_attributes[1].count, 1) + self.assertEqual(custom_attributes[1].min, "NA") + self.assertEqual(custom_attributes[1].max, "NA") + self.assertEqual(custom_attributes[1].average, "NA") + self.assertEqual(custom_attributes[1].median, "NA") + self.assertEqual(custom_attributes[1].stddev, "NA") + self.assertEqual(custom_attributes[1].sum, "NA") + self.assertEqual(custom_attributes[1].unique_values, "NA") + + def test_dataset_bbox(self): + lyr = Dataset.objects.all().first() + dataset_bbox = lyr.bbox[0:4] + logger.debug(dataset_bbox) + + def decimal_encode(bbox): + _bbox = [float(o) for o in bbox] + # Must be in the form : [x0, x1, y0, y1 + return [_bbox[0], _bbox[2], _bbox[1], _bbox[3]] + + from geonode.utils import bbox_to_projection + + projected_bbox = decimal_encode( + bbox_to_projection( + [float(coord) for coord in dataset_bbox] + + [ + lyr.srid, + ], + target_srid=4326, + )[:4] + ) + logger.debug(projected_bbox) + self.assertEqual(projected_bbox, [-180.0, -90.0, 180.0, 90.0]) + logger.debug(lyr.ll_bbox) + self.assertEqual(lyr.ll_bbox, [-180.0, 180.0, -90.0, 90.0, "EPSG:4326"]) + projected_bbox = decimal_encode( + bbox_to_projection( + [float(coord) for coord in dataset_bbox] + + [ + lyr.srid, + ], + target_srid=3857, + )[:4] + ) + solution = [-20037397.023298454, -74299743.40065672, 20037397.02329845, 74299743.40061197] + logger.debug(projected_bbox) + for coord, check in zip(projected_bbox, solution): + self.assertAlmostEqual(coord, check, places=3) + + def test_dataset_attributes_feature_catalogue(self): + """Test layer feature catalogue functionality""" + self.assertTrue(self.client.login(username="admin", password="admin")) + # test a non-existing layer + url = reverse("dataset_feature_catalogue", args=("bad_dataset",)) + response = self.client.get(url) + self.assertEqual(response.status_code, 404) + + # Get the layer to work with + layer = Dataset.objects.all()[3] + url = reverse("dataset_feature_catalogue", args=(layer.alternate,)) + response = self.client.get(url) + self.assertNotEqual(response.status_code, 404) + + def test_dataset_attribute_config(self): + lyr = Dataset.objects.all().first() + attribute_config = lyr.attribute_config() + custom_attributes = attribute_config["getFeatureInfo"] + self.assertEqual(custom_attributes["fields"], ["place_name", "description", "N\xfamero_De_M\xe9dicos"]) + self.assertEqual(custom_attributes["propertyNames"]["description"], "Description") + self.assertEqual(custom_attributes["propertyNames"]["place_name"], "Place Name") + + attributes = Attribute.objects.filter(dataset=lyr) + for _att in attributes: + self.assertEqual(_att.featureinfo_type, "type_property") + + lyr.featureinfo_custom_template = "

          Test HTML

          " + lyr.use_featureinfo_custom_template = True + lyr.save() + attribute_config = lyr.attribute_config() + self.assertTrue("ftInfoTemplate" in attribute_config) + self.assertEqual(attribute_config["ftInfoTemplate"], "

          Test HTML

          ") + lyr.use_featureinfo_custom_template = False + lyr.save() + attribute_config = lyr.attribute_config() + self.assertTrue("ftInfoTemplate" not in attribute_config) + + def test_dataset_styles(self): + lyr = Dataset.objects.all().first() + # There should be a total of 3 styles + self.assertEqual(len(lyr.styles.all()), 4) + # One of the style is the default one + self.assertEqual(lyr.default_style, Style.objects.get(id=lyr.default_style.id)) + + try: + [str(style) for style in lyr.styles.all()] + except UnicodeEncodeError: + self.fail("str of the Style model throws a UnicodeEncodeError with special characters.") + + def test_dataset_links(self): + lyr = Dataset.objects.filter(subtype="vector").first() + self.assertEqual(lyr.subtype, "vector") + + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata") + self.assertIsNotNone(links) + for ll in links: + self.assertEqual(ll.link_type, "metadata") + + _def_link_types = ("data", "image", "original", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS") + Link.objects.filter(resource=lyr.resourcebase_ptr, link_type__in=_def_link_types).delete() + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data") + self.assertIsNotNone(links) + + set_resource_default_links(lyr, lyr) + + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata") + self.assertIsNotNone(links) + for ll in links: + self.assertEqual(ll.link_type, "metadata") + + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data") + self.assertIsNotNone(links) + + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="image") + self.assertIsNotNone(links) + + lyr = Dataset.objects.filter(subtype="raster").first() + self.assertEqual(lyr.subtype, "raster") + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata") + self.assertIsNotNone(links) + for ll in links: + self.assertEqual(ll.link_type, "metadata") + + _def_link_types = ("data", "image", "original", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS") + Link.objects.filter(resource=lyr.resourcebase_ptr, link_type__in=_def_link_types).delete() + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data") + self.assertIsNotNone(links) + + set_resource_default_links(lyr, lyr) + + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata") + self.assertIsNotNone(links) + for ll in links: + self.assertEqual(ll.link_type, "metadata") + + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data") + self.assertIsNotNone(links) + + links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="image") + self.assertIsNotNone(links) + + def test_get_valid_user(self): + # Verify it accepts an admin user + adminuser = get_user_model().objects.get(is_superuser=True) + valid_user = get_valid_user(adminuser) + msg = f'Passed in a valid admin user "{adminuser}" but got "{valid_user}" in return' + assert valid_user.id == adminuser.id, msg + + # Verify it returns a valid user after receiving None + valid_user = get_valid_user(None) + msg = f'Expected valid user after passing None, got "{valid_user}"' + assert isinstance(valid_user, get_user_model()), msg + + newuser = get_user_model().objects.create(username="arieluser") + valid_user = get_valid_user(newuser) + msg = f'Passed in a valid user "{newuser}" but got "{valid_user}" in return' + assert valid_user.id == newuser.id, msg + + valid_user = get_valid_user("arieluser") + msg = 'Passed in a valid user by username "arieluser" but got' f' "{valid_user}" in return' + assert valid_user.username == "arieluser", msg + + nn = get_anonymous_user() + self.assertRaises(GeoNodeException, get_valid_user, nn) + + def test_dataset_type(self): + self.assertEqual(dataset_type("foo.shp"), "vector") + self.assertEqual(dataset_type("foo.SHP"), "vector") + self.assertEqual(dataset_type("foo.sHp"), "vector") + self.assertEqual(dataset_type("foo.tif"), "raster") + self.assertEqual(dataset_type("foo.TIF"), "raster") + self.assertEqual(dataset_type("foo.TiF"), "raster") + self.assertEqual(dataset_type("foo.geotif"), "raster") + self.assertEqual(dataset_type("foo.GEOTIF"), "raster") + self.assertEqual(dataset_type("foo.gEoTiF"), "raster") + self.assertEqual(dataset_type("foo.tiff"), "raster") + self.assertEqual(dataset_type("foo.TIFF"), "raster") + self.assertEqual(dataset_type("foo.TiFf"), "raster") + self.assertEqual(dataset_type("foo.geotiff"), "raster") + self.assertEqual(dataset_type("foo.GEOTIFF"), "raster") + self.assertEqual(dataset_type("foo.gEoTiFf"), "raster") + self.assertEqual(dataset_type("foo.asc"), "raster") + self.assertEqual(dataset_type("foo.ASC"), "raster") + self.assertEqual(dataset_type("foo.AsC"), "raster") + + # basically anything else should produce a GeoNodeException + self.assertRaises(GeoNodeException, lambda: dataset_type("foo.gml")) + + def test_get_files(self): + def generate_files(*extensions): + if extensions[0].lower() != "shp": + return + d = None + expected_files = None + try: + d = mkdtemp() + fnames = [f"foo.{ext}" for ext in extensions] + expected_files = {ext.lower(): fname for ext, fname in zip(extensions, fnames)} + for f in fnames: + path = os.path.join(d, f) + # open and immediately close to create empty file + open(path, "w").close() + finally: + return d, expected_files + + # Check that a well-formed Shapefile has its components all picked up + d = None + _tmpdir = None + try: + d, expected_files = generate_files("shp", "shx", "prj", "dbf") + gotten_files, _tmpdir = get_files(os.path.join(d, "foo.shp")) + gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()} + self.assertEqual(gotten_files, expected_files) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + if _tmpdir is not None: + shutil.rmtree(_tmpdir, ignore_errors=True) + + # Check that a Shapefile missing required components raises an + # exception + d = None + try: + d, expected_files = generate_files("shp", "shx", "prj") + self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp"))) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + + # Check that including an SLD with a valid shapefile results in the SLD + # getting picked up + d = None + _tmpdir = None + try: + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + d, expected_files = generate_files("shp", "shx", "prj", "dbf", "sld") + gotten_files, _tmpdir = get_files(os.path.join(d, "foo.shp")) + gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()} + self.assertEqual(gotten_files, expected_files) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + if _tmpdir is not None: + shutil.rmtree(_tmpdir, ignore_errors=True) + + # Check that capitalized extensions are ok + d = None + _tmpdir = None + try: + d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF") + gotten_files, _tmpdir = get_files(os.path.join(d, "foo.SHP")) + gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()} + self.assertEqual(gotten_files, expected_files) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + if _tmpdir is not None: + shutil.rmtree(_tmpdir, ignore_errors=True) + + # Check that mixed capital and lowercase extensions are ok + d = None + _tmpdir = None + try: + d, expected_files = generate_files("SHP", "shx", "pRJ", "DBF") + gotten_files, _tmpdir = get_files(os.path.join(d, "foo.SHP")) + gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()} + self.assertEqual(gotten_files, expected_files) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + if _tmpdir is not None: + shutil.rmtree(_tmpdir, ignore_errors=True) + + # Check that including both capital and lowercase extensions raises an + # exception + d = None + try: + d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF", "shp", "shx", "prj", "dbf") + + # Only run the tests if this is a case sensitive OS + if len(os.listdir(d)) == len(expected_files): + self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.SHP"))) + self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp"))) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + + # Check that including both capital and lowercase PRJ (this is + # special-cased in the implementation) + d = None + try: + d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF", "prj") + + # Only run the tests if this is a case sensitive OS + if len(os.listdir(d)) == len(expected_files): + self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.SHP"))) + self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp"))) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + + # Check that including both capital and lowercase SLD (this is + # special-cased in the implementation) + d = None + try: + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF", "SLD", "sld") + + # Only run the tests if this is a case sensitive OS + if len(os.listdir(d)) == len(expected_files): + self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.SHP"))) + self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp"))) + finally: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + + def test_get_valid_name(self): + self.assertEqual(get_valid_name("blug"), "blug") + self.assertEqual(get_valid_name("<-->"), "_") + self.assertEqual(get_valid_name(""), "_ab_") + self.assertNotEqual(get_valid_name("CA"), "CA_1") + self.assertNotEqual(get_valid_name("CA"), "CA_1") + + def test_get_valid_dataset_name(self): + self.assertEqual(get_valid_dataset_name("blug", False), "blug") + self.assertEqual(get_valid_dataset_name("blug", True), "blug") + + self.assertEqual(get_valid_dataset_name("", False), "_ab_") + self.assertEqual(get_valid_dataset_name("", True), "") + + self.assertEqual(get_valid_dataset_name("<-->", False), "_") + self.assertEqual(get_valid_dataset_name("<-->", True), "<-->") + + self.assertNotEqual(get_valid_dataset_name("CA", False), "CA_1") + self.assertNotEqual(get_valid_dataset_name("CA", False), "CA_1") + self.assertEqual(get_valid_dataset_name("CA", True), "CA") + self.assertEqual(get_valid_dataset_name("CA", True), "CA") + + layer = Dataset.objects.get(name="CA") + self.assertNotEqual(get_valid_dataset_name(layer, False), "CA_1") + self.assertEqual(get_valid_dataset_name(layer, True), "CA") + + self.assertRaises(GeoNodeException, get_valid_dataset_name, 12, False) + self.assertRaises(GeoNodeException, get_valid_dataset_name, 12, True) + + # NOTE: we don't care about file content for many of these tests (the + # forms under test validate based only on file name, and leave actual + # content inspection to GeoServer) but Django's form validation will omit + # any files with empty bodies. + # + # That is, this leads to mysterious test failures: + # SimpleUploadedFile('foo', ' '.encode("UTF-8")) + # + # And this should be used instead to avoid that: + # SimpleUploadedFile('foo', ' '.encode("UTF-8")) + + def testJSONField(self): + field = JSONField() + # a valid JSON document should pass + field.clean('{ "users": [] }') + + # text which is not JSON should fail + self.assertRaises(ValidationError, lambda: field.clean("")) + + def test_sld_upload(self): + """Test layer remove functionality""" + layer = Dataset.objects.all().first() + url = reverse("dataset_sld_upload", args=(layer.alternate,)) + # Now test with a valid user + self.client.login(username="admin", password="admin") + + # test a method other than POST and GET + response = self.client.put(url) + content = response.content.decode("utf-8") + self.assertEqual(response.status_code, 200) + self.assertFalse("#modal_perms" in content) + + def test_category_counts(self): + topics = TopicCategory.objects.all() + topics = topics.annotate(**{"dataset_count": Count("resourcebase__dataset__category")}) + location = topics.get(identifier="location") + # there are three layers with location category + self.assertEqual(location.dataset_count, 3) + + # change the category of one layers_count + layer = Dataset.objects.filter(category=location)[0] + elevation = topics.get(identifier="elevation") + layer.category = elevation + layer.save() + + # reload the categories since it's caching the old count + topics = topics.annotate(**{"dataset_count": Count("resourcebase__dataset__category")}) + location = topics.get(identifier="location") + elevation = topics.get(identifier="elevation") + self.assertEqual(location.dataset_count, 2) + self.assertEqual(elevation.dataset_count, 4) + + # delete a layer and check the count update + # use the first since it's the only one which has styles + layer = Dataset.objects.all().first() + elevation = topics.get(identifier="elevation") + self.assertEqual(elevation.dataset_count, 4) + layer.delete() + topics = topics.annotate(**{"dataset_count": Count("resourcebase__dataset__category")}) + elevation = topics.get(identifier="elevation") + self.assertEqual(elevation.dataset_count, 3) + + def test_assign_change_dataset_data_perm(self): + """ + Ensure set_permissions supports the change_dataset_data permission. + """ + layer = Dataset.objects.first() + user = get_anonymous_user() + layer.set_permissions({"users": {user.username: ["change_dataset_data"]}}) + perms = layer.get_all_level_info() + self.assertNotIn(user, perms["users"]) + self.assertNotIn(user.username, perms["users"]) + + def test_batch_edit(self): + """ + Test batch editing of metadata fields. + """ + Model = Dataset + view = "dataset_batch_metadata" + resources = Model.objects.all()[:3] + ids = ",".join(str(element.pk) for element in resources) + # test non-admin access + self.client.login(username="bobby", password="bob") + response = self.client.get(reverse(view)) + self.assertTrue(response.status_code in (401, 403)) + # test group change + group = Group.objects.first() + self.client.login(username="admin", password="admin") + response = self.client.post( + reverse(view), + data={"group": group.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.group, group) + # test owner change + owner = get_user_model().objects.first() + response = self.client.post( + reverse(view), + data={"owner": owner.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.owner, owner) + # test license change + license = License.objects.first() + response = self.client.post( + reverse(view), + data={"license": license.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.license, license) + # test regions change + region = Region.objects.first() + response = self.client.post( + reverse(view), + data={"region": region.pk, "ids": ids, "regions": 1}, + ) + self.assertEqual(response.status_code, 302) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + if resource.regions.all(): + self.assertTrue(region in resource.regions.all()) + # test language change + language = "eng" + response = self.client.post( + reverse(view), + data={"language": language, "ids": ids, "regions": 1}, + ) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + self.assertEqual(resource.language, language) + # test keywords change + keywords = "some,thing,new" + response = self.client.post( + reverse(view), + data={"keywords": keywords, "ids": ids, "regions": 1}, + ) + resources = Model.objects.filter(id__in=[r.pk for r in resources]) + for resource in resources: + for word in resource.keywords.all(): + self.assertTrue(word.name in keywords.split(",")) + + def test_surrogate_escape_string(self): + surrogate_escape_raw = "Zo\udcc3\udcab" + surrogate_escape_expected = "Zoë" + surrogate_escape_result = surrogate_escape_string( + surrogate_escape_raw, "UTF-8" + ) # add more test cases using different charsets? + self.assertEqual( + surrogate_escape_result, + surrogate_escape_expected, + "layers.utils.surrogate_escape_string did not produce expected result. " + f"Expected {surrogate_escape_expected}, received {surrogate_escape_result}", + ) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_assign_remove_permissions(self): + # Assing + layer = Dataset.objects.all().first() + perm_spec = layer.get_all_level_info() + self.assertNotIn(get_user_model().objects.get(username="norman"), perm_spec["users"]) + + utils.set_datasets_permissions( + "edit", resources_names=[layer.name], users_usernames=["norman"], delete_flag=False, verbose=True + ) + perm_spec = layer.get_all_level_info() + _c = 0 + if "users" in perm_spec: + for _u in perm_spec["users"]: + if _u == "norman" or _u == get_user_model().objects.get(username="norman"): + _c += 1 + # "norman" has both read & write permissions + self.assertEqual(_c, 1) + + # Remove + utils.set_datasets_permissions( + "read", resources_names=[layer.name], users_usernames=["norman"], delete_flag=True, verbose=True + ) + perm_spec = layer.get_all_level_info() + _c = 0 + if "users" in perm_spec: + for _u in perm_spec["users"]: + if _u == "norman" or _u == get_user_model().objects.get(username="norman"): + _c += 1 + # "norman" has no permissions + self.assertEqual(_c, 0) + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_assign_remove_permissions_for_groups(self): + # Assing + layer = Dataset.objects.all().first() + perm_spec = layer.get_all_level_info() + group_profile = GroupProfile.objects.create(slug="group1", title="group1", access="public") + self.assertNotIn(group_profile, perm_spec["groups"]) + + # giving manage permissions to the group + utils.set_datasets_permissions( + "manage", resources_names=[layer.name], groups_names=["group1"], delete_flag=False, verbose=True + ) + perm_spec = layer.get_all_level_info() + expected = { + "change_dataset_data", + "change_dataset_style", + "change_resourcebase", + "change_resourcebase_metadata", + "change_resourcebase_permissions", + "delete_resourcebase", + "download_resourcebase", + "publish_resourcebase", + "view_resourcebase", + } + # checking the perms list + self.assertSetEqual(expected, set(perm_spec["groups"][group_profile.group])) + + # Chaning perms to the group from manage to read + utils.set_datasets_permissions( + "view", resources_names=[layer.name], groups_names=["group1"], delete_flag=False, verbose=True + ) + perm_spec = layer.get_all_level_info() + expected = {"view_resourcebase"} + # checking the perms list + self.assertSetEqual(expected, set(perm_spec["groups"][group_profile.group])) + + # Chaning perms to the group from manage to read + utils.set_datasets_permissions( + "view", resources_names=[layer.name], groups_names=["group1"], delete_flag=True, verbose=True + ) + perm_spec = layer.get_all_level_info() + # checking the perms list + self.assertTrue(group_profile.group not in perm_spec["groups"]) + + if group_profile: + group_profile.delete() + + def test_dataset_download_not_found_for_non_existing_dataset(self): + self.client.login(username="admin", password="admin") + url = reverse("dataset_download", args=["foo-dataset"]) + response = self.client.get(url) + self.assertEqual(404, response.status_code) + + @override_settings(USE_GEOSERVER=False) + def test_dataset_download_redirect_to_proxy_url(self): + # if settings.USE_GEOSERVER is false, the URL must be redirected + self.client.login(username="admin", password="admin") + dataset = Dataset.objects.first() + url = reverse("dataset_download", args=[dataset.alternate]) + response = self.client.get(url) + self.assertEqual(302, response.status_code) + self.assertEqual(f"/download/{dataset.id}", response.url) + + def test_dataset_download_invalid_wps_format(self): + # if settings.USE_GEOSERVER is false, the URL must be redirected + self.client.login(username="admin", password="admin") + dataset = Dataset.objects.first() + url = reverse("dataset_download", args=[dataset.alternate]) + response = self.client.get(f"{url}?export_format=foo") + self.assertEqual(500, response.status_code) + self.assertDictEqual({"error": "The format provided is not valid for the selected resource"}, response.json()) + + @patch("geonode.layers.download_handler.HttpClient.request") + def test_dataset_download_call_the_catalog_raise_error_for_no_200(self, mocked_catalog): + _response = MagicMock(status_code=500, content="foo-bar") + mocked_catalog.return_value = _response, "foo-bar" + # if settings.USE_GEOSERVER is false, the URL must be redirected + self.client.login(username="admin", password="admin") + dataset = Dataset.objects.first() + url = reverse("dataset_download", args=[dataset.alternate]) + response = self.client.get(url) + self.assertEqual(500, response.status_code) + self.assertDictEqual({"error": "Download dataset exception: error during call with GeoServer"}, response.json()) + + def test_dataset_download_call_the_catalog_raise_error_for_error_content(self): + content = """ + + + Foo Bar Exception + + + """ # noqa + _response = MagicMock(status_code=200, text=content, headers={"Content-Type": "text/xml"}) + # if settings.USE_GEOSERVER is false, the URL must be redirected + self.client.login(username="admin", password="admin") + dataset = Dataset.objects.first() + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + mocked_catalog.return_value = _response, content + url = reverse("dataset_download", args=[dataset.alternate]) + response = self.client.get(url) + self.assertEqual(500, response.status_code) + self.assertDictEqual({"error": "InvalidParameterValue: Foo Bar Exception"}, response.json()) + + def test_dataset_download_call_the_catalog_works(self): + # if settings.USE_GEOSERVER is false, the URL must be redirected + _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa + self.client.login(username="admin", password="admin") + dataset = Dataset.objects.first() + layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + mocked_catalog.return_value = _response, "" + url = reverse("dataset_download", args=[layer.alternate]) + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + + def test_dataset_download_call_the_catalog_not_work_without_download_resurcebase_perm(self): + dataset = Dataset.objects.first() + dataset.set_permissions({"users": {"bobby": ["base.view_resourcebase"]}}) + self.client.login(username="bobby", password="bob") + url = reverse("dataset_download", args=[dataset.alternate]) + response = self.client.get(url) + self.assertEqual(404, response.status_code) + + def test_dataset_download_call_the_catalog_work_anonymous(self): + # if settings.USE_GEOSERVER is false, the URL must be redirected + _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa + dataset = Dataset.objects.first() + layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + mocked_catalog.return_value = _response, "" + url = reverse("dataset_download", args=[layer.alternate]) + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + + @override_settings(USE_GEOSERVER=True) + @patch("geonode.layers.download_handler.get_template") + def test_dataset_download_call_the_catalog_work_for_raster(self, pathed_template): + # if settings.USE_GEOSERVER is false, the URL must be redirected + _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa + dataset = Dataset.objects.filter(subtype="raster").first() + layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") + Dataset.objects.filter(alternate=layer.alternate).update(subtype="raster") + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + mocked_catalog.return_value = _response, "" + url = reverse("dataset_download", args=[layer.alternate]) + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + """ + Evaluate that the context used by the template contains the right mimetype for the resource + """ + self.assertTupleEqual( + ({"alternate": layer.alternate, "download_format": "image/tiff"},), pathed_template.mock_calls[1].args + ) + + @override_settings(USE_GEOSERVER=True) + @patch("geonode.layers.download_handler.get_template") + def test_dataset_download_call_the_catalog_work_for_vector(self, pathed_template): + # if settings.USE_GEOSERVER is false, the URL must be redirected + _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa + dataset = Dataset.objects.filter(subtype="vector").first() + layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + mocked_catalog.return_value = _response, "" + url = reverse("dataset_download", args=[layer.alternate]) + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + """ + Evaluate that the context used by the template contains the right mimetype for the resource + """ + self.assertTupleEqual( + ({"alternate": layer.alternate, "download_format": "application/zip"},), pathed_template.mock_calls[1].args + ) + + +class TestLayerDetailMapViewRights(GeoNodeBaseTestSupport): + fixtures = ["initial_data.json", "group_test_data.json", "default_oauth_apps.json"] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + create_single_dataset("single_point") + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.user = get_user_model().objects.create(username="dybala", email="dybala@gmail.com") + self.user.set_password("very-secret") + self.admin = get_user_model().objects.get(username="admin") + self.map = Map.objects.create(uuid=str(uuid4()), owner=self.admin, title="test", is_approved=True) + self.not_admin = get_user_model().objects.create(username="r-lukaku", is_active=True) + self.not_admin.set_password("very-secret") + self.not_admin.save() + + self.layer = Dataset.objects.all().first() + create_dataset_data(self.layer.resourcebase_ptr_id) + with DisableDjangoSignals(): + self.map_dataset = MapLayer.objects.create( + name=self.layer.alternate, + map=self.map, + ) + + def test_that_keyword_multiselect_is_disabled_for_non_admin_users(self): + """ + Test that keyword multiselect widget is disabled when the user is not an admin + """ + self.test_dataset = resource_manager.create( + None, resource_type=Dataset, defaults=dict(owner=self.not_admin, title="test", is_approved=True) + ) + + url = reverse("dataset_metadata", args=(self.test_dataset.alternate,)) + self.client.login(username=self.not_admin.username, password="very-secret") + with self.settings(FREETEXT_KEYWORDS_READONLY=True): + response = self.client.get(url) + self.assertTrue(response.context["form"]["keywords"].field.disabled, self.test_dataset.alternate) + + def test_that_keyword_multiselect_is_not_disabled_for_admin_users(self): + """ + Test that only admin users can create/edit keywords when FREETEXT_KEYWORDS_READONLY=True + """ + admin = self.not_admin + admin.is_superuser = True + admin.save() + + self.test_dataset = resource_manager.create( + None, resource_type=Dataset, defaults=dict(owner=admin, title="test", is_approved=True) + ) + + url = reverse("dataset_metadata", args=(self.test_dataset.alternate,)) + + self.client.login(username=admin.username, password="very-secret") + with self.settings(FREETEXT_KEYWORDS_READONLY=True): + response = self.client.get(url) + self.assertFalse(response.context["form"]["keywords"].field.disabled, self.test_dataset.alternate) + + def test_that_featured_enabling_and_disabling_for_users(self): + self.test_dataset = resource_manager.create( + None, resource_type=Dataset, defaults=dict(owner=self.not_admin, title="test", is_approved=True) + ) + + url = reverse("dataset_metadata", args=(self.test_dataset.alternate,)) + # Non Admins + self.client.login(username=self.not_admin.username, password="very-secret") + response = self.client.get(url) + self.assertFalse(self.not_admin.is_superuser) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.context["form"]["featured"].field.disabled) + # Admin + self.client.login(username="admin", password="admin") + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertFalse(response.context["form"]["featured"].field.disabled) + + def test_that_non_admin_user_cannot_create_edit_keyword(self): + """ + Test that non admin users cannot edit/create keywords when FREETEXT_KEYWORDS_READONLY=True + """ + self.test_dataset = resource_manager.create( + None, resource_type=Dataset, defaults=dict(owner=self.not_admin, title="test", is_approved=True) + ) + + url = reverse("dataset_metadata", args=(self.test_dataset.alternate,)) + self.client.login(username=self.not_admin.username, password="very-secret") + with self.settings(FREETEXT_KEYWORDS_READONLY=True): + response = self.client.post(url, data={"resource-keywords": "wonderful-keyword"}) + self.assertEqual(response.status_code, 401) + self.assertEqual(response.content, b"Unauthorized: Cannot edit/create Free-text Keywords") + + def test_that_keyword_multiselect_is_enabled_for_non_admin_users_when_freetext_keywords_readonly_istrue(self): + """ + Test that keyword multiselect widget is not disabled when the user is not an admin + and FREETEXT_KEYWORDS_READONLY=False + """ + self.test_dataset = resource_manager.create( + None, resource_type=Dataset, defaults=dict(owner=self.not_admin, title="test", is_approved=True) + ) + + url = reverse("dataset_metadata", args=(self.test_dataset.alternate,)) + + self.client.login(username=self.not_admin.username, password="very-secret") + with self.settings(FREETEXT_KEYWORDS_READONLY=False): + response = self.client.get(url) + self.assertFalse(response.context["form"]["keywords"].field.disabled, self.test_dataset.alternate) + + response = self.client.get(reverse("dataset_embed", args=(self.layer.alternate,))) + self.assertIsNotNone(response.context["resource"]) + + def test_that_only_users_with_permissions_can_view_maps_in_dataset_view(self): + """ + Test only users with view permissions to a map can view them in layer detail view + """ + resource_manager.remove_permissions(self.map.uuid, instance=self.map.get_self_resource()) + self.client.login(username="admin", password="admin") + response = self.client.get(reverse("dataset_embed", args=(self.layer.alternate,))) + self.assertEqual(response.context["resource"].alternate, self.map_dataset.name) + + def test_update_with_a_comma_in_title_is_replaced_by_undescore(self): + """ + Test that when changing the dataset title, if the entered title has a comma it is replaced by an undescore. + """ + self.test_dataset = None + try: + self.test_dataset = resource_manager.create( + None, resource_type=Dataset, defaults=dict(owner=self.not_admin, title="test", is_approved=True) + ) + + data = { + "resource-title": "test,comma,2021", + "resource-owner": self.test_dataset.owner.id, + "resource-date": str(self.test_dataset.date), + "resource-date_type": self.test_dataset.date_type, + "resource-language": self.test_dataset.language, + "dataset_attribute_set-TOTAL_FORMS": 0, + "dataset_attribute_set-INITIAL_FORMS": 0, + } + + url = reverse("dataset_metadata", args=(self.test_dataset.alternate,)) + self.client.login(username=self.not_admin.username, password="very-secret") + response = self.client.post(url, data=data) + self.test_dataset.refresh_from_db() + self.assertEqual(self.test_dataset.title, "test_comma_2021") + self.assertEqual(response.status_code, 200) + finally: + if self.test_dataset: + self.test_dataset.delete() + + +class LayerNotificationsTestCase(NotificationsTestsHelper): + type = "dataset" + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.user = "admin" + self.passwd = "admin" + self.anonymous_user = get_anonymous_user() + self.u = get_user_model().objects.get(username=self.user) + self.u.email = "test@email.com" + self.u.is_active = True + self.u.save() + self.setup_notifications_for(DatasetAppConfig.NOTIFICATIONS, self.u) + self.norman = get_user_model().objects.get(username="norman") + self.norman.email = "norman@email.com" + self.norman.is_active = True + self.norman.save() + self.setup_notifications_for(DatasetAppConfig.NOTIFICATIONS, self.norman) + + def testLayerNotifications(self): + with self.settings( + EMAIL_ENABLE=True, + NOTIFICATION_ENABLED=True, + NOTIFICATIONS_BACKEND="pinax.notifications.backends.email.EmailBackend", + PINAX_NOTIFICATIONS_QUEUE_ALL=False, + ): + self.clear_notifications_queue() + self.client.login(username=self.user, password=self.passwd) + + _l = resource_manager.create( + None, + resource_type=Dataset, + defaults=dict( + name="test notifications", + title="test notifications", + bbox_polygon=Polygon.from_bbox((-180, -90, 180, 90)), + srid="EPSG:4326", + owner=self.norman, + ), + ) + + self.assertTrue(self.check_notification_out("dataset_created", self.u)) + # Ensure "resource.owner" won't be notified for having uploaded its own resource + self.assertFalse(self.check_notification_out("dataset_created", self.norman)) + + self.clear_notifications_queue() + _l.name = "test notifications 2" + _l.save(notify=True) + self.assertTrue(self.check_notification_out("dataset_updated", self.u)) + + self.clear_notifications_queue() + + +""" +Smoke test to explain how the uuidhandler will override the uuid for the layers +Documentation of the handler is available here: +https://github.com/GeoNode/documentation/blob/703cc6ba92b7b7a83637a874fb449420a9f8b78a/basic/settings/index.rst#uuid-handler +""" + + +class DummyUUIDHandler: + def __init__(self, instance): + self.instance = instance + + def create_uuid(self): + return f"abc:{self.instance.uuid}" + + +class TestCustomUUidHandler(TestCase): + def setUp(self): + User = get_user_model() + self.user = User.objects.create(username="test", email="test@test.com") + self.sut = Dataset.objects.create( + name="testLayer", owner=self.user, title="test", is_approved=True, uuid="abc-1234-abc" + ) + + def test_dataset_will_maintain_his_uud_if_no_handler_is_definded(self): + expected = "abc-1234-abc" + self.assertEqual(expected, self.sut.uuid) + + @override_settings(LAYER_UUID_HANDLER="geonode.layers.tests.DummyUUIDHandler") + def test_dataset_will_override_the_uuid_if_handler_is_defined(self): + resource_manager.update(None, instance=self.sut, keywords=["updating", "values"]) + expected = "abc:abc-1234-abc" + actual = Dataset.objects.get(id=self.sut.id) + self.assertEqual(expected, actual.uuid) + + self.assertIsNotNone(self.sut.ows_url) + self.assertIsNotNone(self.sut.ptype) + self.assertIsNotNone(self.sut.sourcetype) + + +class TestSetMetadata(TestCase): + def setUp(self): + self.maxDiff = None + self.invalid_xml = "xml" + self.exml_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" + self.custom = [ + { + "keywords": ["features", "test_dataset"], + "thesaurus": {"date": None, "datetype": None, "title": None}, + "type": "theme", + }, + { + "keywords": ["no conditions to access and use"], + "thesaurus": { + "date": "2020-10-30T16:58:34", + "datetype": "publication", + "title": "Test for ordering", + }, + "type": None, + }, + { + "keywords": ["ad", "af"], + "thesaurus": { + "date": "2008-06-01", + "datetype": "publication", + "title": "GEMET - INSPIRE themes, version 1.0", + }, + "type": None, + }, + {"keywords": ["Global"], "thesaurus": {"date": None, "datetype": None, "title": None}, "type": "place"}, + ] + + def test_set_metadata_will_rase_an_exception_if_is_not_valid_xml(self): + with self.assertRaises(GeoNodeException): + set_metadata(self.invalid_xml) + + def test_set_metadata_return_expected_values_from_xml(self): + import datetime + + identifier, vals, regions, keywords, _ = set_metadata(open(self.exml_path).read()) + expected_vals = { + "abstract": "real abstract", + "constraints_other": "Not Specified: The original author did not specify a license.", + "data_quality_statement": "Created with GeoNode", + "date": datetime.datetime(2021, 4, 9, 9, 0, 46), + "language": "eng", + "purpose": None, + "spatial_representation_type": "vector", + "supplemental_information": "No information provided", + "temporal_extent_end": None, + "temporal_extent_start": None, + "title": "test_dataset", + } + expected_keywords = [] + for kw in [kw.get("keywords") for kw in self.custom if kw["type"] != "place"]: + for _kw in [_kw for _kw in kw]: + expected_keywords.append(_kw) + self.assertEqual("7cfbc42c-efa7-431c-8daa-1399dff4cd19", identifier) + self.assertListEqual(["Global"], regions) + self.assertDictEqual(expected_vals, vals) + self.assertListEqual(expected_keywords, keywords) + + def test_convert_keyword_should_empty_list_for_empty_keyword(self): + actual = convert_keyword([]) + self.assertListEqual([], actual) + + def test_convert_keyword_should_empty_list_for_non_empty_keyword(self): + expected = [ + { + "keywords": ["abc"], + "thesaurus": {"date": None, "datetype": None, "title": None}, + "type": "theme", + } + ] + actual = convert_keyword(["abc"]) + self.assertListEqual(expected, actual) + + +""" +Smoke test to explain how the new function for multiple parsers will work +Is required to define a fuction that takes 1 parameters (the metadata xml) and return 4 parameters. + Parameters: + xml (str): TextIOWrapper read example: open(self.exml_path).read()) + + Returns: + Tuple (tuple): + - (identifier, vals, regions, keywords) + + identifier(str): default empy, + vals(dict): default empty, + regions(list): default empty, + keywords(list(dict)): default empty +""" + + +class TestCustomMetadataParser(TestCase): + def setUp(self): + import datetime + + self.exml_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" + self.expected_vals = { + "abstract": "real abstract", + "constraints_other": "Not Specified: The original author did not specify a license.", + "data_quality_statement": "Created with GeoNode", + "date": datetime.datetime(2021, 4, 9, 9, 0, 46), + "language": "eng", + "purpose": None, + "spatial_representation_type": "vector", + "supplemental_information": "No information provided", + "temporal_extent_end": None, + "temporal_extent_start": None, + "title": "test_dataset", + } + + self.keywords = [ + { + "keywords": ["features", "test_dataset"], + "thesaurus": {"date": None, "datetype": None, "title": None}, + "type": "theme", + }, + { + "keywords": ["no conditions to access and use"], + "thesaurus": { + "date": "2020-10-30T16:58:34", + "datetype": "publication", + "title": "Test for ordering", + }, + "type": None, + }, + { + "keywords": ["ad", "af"], + "thesaurus": { + "date": "2008-06-01", + "datetype": "publication", + "title": "GEMET - INSPIRE themes, version 1.0", + }, + "type": None, + }, + {"keywords": ["Global"], "thesaurus": {"date": None, "datetype": None, "title": None}, "type": "place"}, + ] + + def test_will_use_only_the_default_metadata_parser(self): + identifier, vals, regions, keywords, _ = parse_metadata(open(self.exml_path).read()) + expected_keywords = [] + for kw in [kw.get("keywords") for kw in self.keywords if kw["type"] != "place"]: + for _kw in [_kw for _kw in kw]: + expected_keywords.append(_kw) + self.assertEqual("7cfbc42c-efa7-431c-8daa-1399dff4cd19", identifier) + self.assertListEqual(["Global"], regions) + self.assertListEqual(expected_keywords, keywords) + self.assertDictEqual(self.expected_vals, vals) + + @override_settings(METADATA_PARSERS=["__DEFAULT__", "geonode.layers.tests.dummy_metadata_parser"]) + def test_will_use_both_parsers_defined(self): + identifier, vals, regions, keywords, _ = parse_metadata(open(self.exml_path).read()) + self.assertEqual("7cfbc42c-efa7-431c-8daa-1399dff4cd19", identifier) + self.assertListEqual(["Global", "Europe"], regions) + self.assertEqual("Passed through new parser", keywords) + self.assertDictEqual(self.expected_vals, vals) + + def test_convert_keyword_should_empty_list_for_empty_keyword(self): + actual = convert_keyword([]) + self.assertListEqual([], actual) + + def test_convert_keyword_should_non_empty_list_for_empty_keyword(self): + expected = [ + { + "keywords": ["abc"], + "thesaurus": {"date": None, "datetype": None, "title": None}, + "type": "theme", + } + ] + actual = convert_keyword(["abc"]) + self.assertListEqual(expected, actual) + + +""" +Just a dummy function required for the smoke test above +""" + + +def dummy_metadata_parser(exml, uuid, vals, regions, keywords, custom): + keywords = "Passed through new parser" + regions.append("Europe") + return uuid, vals, regions, keywords, custom + + +class TestDatasetForm(GeoNodeBaseTestSupport): + def setUp(self) -> None: + self.user = get_user_model().objects.get(username="admin") + self.user2 = get_user_model().objects.get_or_create(username="svenzwei") + + self.dataset = create_single_dataset("my_single_layer", owner=self.user) + self.sut = DatasetForm + self.time_form = DatasetTimeSerieForm + + def test_resource_form_is_invalid_extra_metadata_not_json_format(self): + self.client.login(username="admin", password="admin") + url = reverse("dataset_metadata", args=(self.dataset.alternate,)) + response = self.client.post( + url, + data={ + "resource-owner": self.dataset.owner.id, + "resource-title": "layer_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "not-a-json", + }, + ) + expected = { + "success": False, + "errors": ["extra_metadata: The value provided for the Extra metadata field is not a valid JSON"], + } + self.assertDictEqual(expected, response.json()) + + def test_change_owner_in_metadata(self): + try: + test_user = get_user_model().objects.create_user( + username="non_auth", email="non_auth@geonode.org", password="password" + ) + norman = get_user_model().objects.get(username="norman") + dataset = Dataset.objects.first() + data = { + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "dataset_attribute_set-TOTAL_FORMS": 0, + "dataset_attribute_set-INITIAL_FORMS": 0, + } + perm_spec = { + "users": { + "non_auth": [ + "change_resourcebase_metadata", + "change_resourcebase", + ], + "norman": ["change_resourcebase_metadata", "change_resourcebase_permissions"], + } + } + self.assertTrue(dataset.set_permissions(perm_spec)) + self.assertFalse(test_user.has_perm("change_resourcebase_permissions", dataset.get_self_resource())) + + url = reverse("dataset_metadata", args=(dataset.alternate,)) + # post as non-authorised user + self.client.login(username="non_auth", password="password") + data["resource-owner"] = test_user.id + response = self.client.post(url, data=data) + self.assertEqual(response.status_code, 200) + self.assertNotEqual(dataset.owner, test_user) + # post as admin + self.client.login(username="admin", password="admin") + response = self.client.post(url, data=data) + dataset.refresh_from_db() + self.assertEqual(response.status_code, 200) + self.assertEqual(dataset.owner, test_user) + # post as an authorised user + self.client.login(username="norman", password="norman") + self.assertTrue(norman.has_perm("change_resourcebase_permissions", dataset.get_self_resource())) + data["resource-owner"] = norman.id + response = self.client.post(url, data=data) + dataset.refresh_from_db() + self.assertEqual(response.status_code, 200) + self.assertEqual(dataset.owner, norman) + finally: + get_user_model().objects.filter(username="non_auth").delete + Dataset.objects.filter(name="dataset_name").delete() + + @override_settings(EXTRA_METADATA_SCHEMA={"key": "value"}) + def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): + self.client.login(username="admin", password="admin") + url = reverse("dataset_metadata", args=(self.dataset.alternate,)) + response = self.client.post( + url, + data={ + "resource-owner": self.dataset.owner.id, + "resource-title": "layer_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "[{'key': 'value'}]", + }, + ) + expected = { + "success": False, + "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource dataset"], + } + self.assertDictEqual(expected, response.json()) + + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): + self.client.login(username="admin", password="admin") + url = reverse("dataset_metadata", args=(self.dataset.alternate,)) + response = self.client.post( + url, + data={ + "resource-owner": self.dataset.owner.id, + "resource-title": "layer_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": '[{"key": "value"},{"id": "int", "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + ) + expected = ( + "extra_metadata: Missing keys: 'field_label', 'field_name', 'field_value', 'filter_header' at index 0 " + ) + self.assertIn(expected, response.json()["errors"][0]) + + def test_resource_form_is_valid_extra_metadata(self): + form = self.sut( + instance=self.dataset, + data={ + "owner": self.dataset.owner.id, + "title": "layer_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + ) + self.assertTrue(form.is_valid()) + + def test_dataset_time_form_should_work(self): + attr, _ = Attribute.objects.get_or_create( + dataset=self.dataset, attribute="field_date", attribute_type="xsd:dateTime" + ) + self.dataset.attribute_set.add(attr) + self.dataset.save() + form = self.time_form( + instance=self.dataset, + data={ + "attribute": self.dataset.attributes.first().id, + "end_attribute": "", + "presentation": "DISCRETE_INTERVAL", + "precision_value": 12345, + "precision_step": "seconds", + }, + ) + self.assertTrue(form.is_valid()) + self.assertDictEqual({}, form.errors) + + def test_dataset_time_form_should_work_with_date_attribute(self): + attr, _ = Attribute.objects.get_or_create( + dataset=self.dataset, attribute="field_date", attribute_type="xsd:date" + ) + self.dataset.attribute_set.add(attr) + self.dataset.save() + form = self.time_form( + instance=self.dataset, + data={ + "attribute": self.dataset.attributes.first().id, + "end_attribute": "", + "presentation": "DISCRETE_INTERVAL", + "precision_value": 12345, + "precision_step": "seconds", + }, + ) + self.assertTrue(form.is_valid()) + self.assertDictEqual({}, form.errors) + expected_choises = [(None, "-----"), (self.dataset.attributes.first().id, "field_date")] + actual_choices = form.fields.get("attribute").choices + self.assertListEqual(expected_choises, actual_choices) + + def test_timeserie_raise_error_if_not_valid_attribute(self): + attr, _ = Attribute.objects.get_or_create( + dataset=self.dataset, attribute="field_date", attribute_type="xsd:string" + ) + self.dataset.attribute_set.add(attr) + self.dataset.save() + form = self.time_form( + instance=self.dataset, + data={ + "attribute": self.dataset.attributes.first().id, + "end_attribute": "", + "presentation": "DISCRETE_INTERVAL", + "precision_value": 12345, + "precision_step": "seconds", + }, + ) + self.assertFalse(form.is_valid()) + self.assertEqual( + f"Select a valid choice. {self.dataset.attributes.first().id} is not one of the available choices.", + form.errors.get("attribute")[0], + ) + expected_choises = [(None, "-----")] + actual_choices = form.fields.get("attribute").choices + self.assertListEqual(expected_choises, actual_choices) + + def test_dataset_time_form_should_raise_error_if_invalid_payload(self): + attr, _ = Attribute.objects.get_or_create( + dataset=self.dataset, attribute="field_date", attribute_type="xsd:dateTime" + ) + self.dataset.attribute_set.add(attr) + self.dataset.save() + form = self.time_form( + instance=self.dataset, + data={ + "attribute": self.dataset.attributes.first().id, + "end_attribute": "", + "presentation": "INVALID_PRESENTATION_VALUE", + "precision_value": 12345, + "precision_step": "seconds", + }, + ) + self.assertFalse(form.is_valid()) + self.assertTrue("presentation" in form.errors) + self.assertEqual( + "Select a valid choice. INVALID_PRESENTATION_VALUE is not one of the available choices.", + form.errors["presentation"][0], + ) + + def test_resource_form_is_valid_single_user_contact_role(self): + """test if passing a single user to a contact role form is working""" + users = get_user_model().objects.filter(username="svenzwei") + cr = Roles.get_multivalue_ones()[0] + form = self.sut( + instance=self.dataset, + data={ + "owner": self.dataset.owner.id, + cr.name: [u.username for u in users], + "title": "layer_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + ) + self.assertTrue(form.is_valid()) + self.assertEqual(list(form.cleaned_data[cr.name]), list(users)) + + def test_resource_form_is_valid_multiple_user_contact_role_as_queryset(self): + """test if passing a multiple user to a contact role form is working""" + users = get_user_model().objects.filter(username__in=["svenzwei", "admin"]) + for cr in Roles.get_multivalue_ones(): + form = self.sut( + instance=self.dataset, + data={ + "owner": self.dataset.owner.id, + cr.name: [u.username for u in users], + "title": "layer_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]', + }, + ) + self.assertTrue(form.is_valid()) + self.assertEqual(list(form.cleaned_data[cr.name]), list(users)) + + def test_resource_form_is_invalid_with_incompleted_timeserie_data(self): + self.client.login(username="admin", password="admin") + url = reverse("dataset_metadata", args=(self.dataset.alternate,)) + response = self.client.post( + url, + data={ + "resource-owner": self.dataset.owner.id, + "resource-title": "layer_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-has_time": True, + "dataset_attribute_set-TOTAL_FORMS": 0, + "dataset_attribute_set-INITIAL_FORMS": 0, + }, + ) + expected = { + "success": False, + "errors": [ + "The Timeseries configuration is invalid. Please select at least one option between the `attribute` and `end_attribute`, otherwise remove the 'has_time' flag" + ], + } + self.assertDictEqual(expected, response.json()) + + +class SetLayersPermissionsCommand(GeoNodeBaseTestSupport): + """ + Unittest to ensure that the management command "set_layers_permissions" + behaves as expected + """ + + def test_user_get_the_download_permissions_for_the_selected_dataset(self): + """ + Given a user, the compact perms and the resource id, it shoul set the + permissions for the selected resource + """ + try: + expected_perms = {"view_resourcebase", "download_resourcebase"} + + dataset, args, username, opts = self._create_arguments(perms_type="download") + call_command("set_layers_permissions", *args, **opts) + + self._assert_perms(expected_perms, dataset, username) + finally: + if dataset: + dataset.delete() + + def test_user_get_the_view_permissions_for_the_selected_dataset(self): + """ + Given a user, the compact perms and the resource id, it shoul set the + permissions for the selected resource + """ + try: + expected_perms = {"view_resourcebase"} + dataset, args, username, opts = self._create_arguments(perms_type="view") + + call_command("set_layers_permissions", *args, **opts) + + self._assert_perms(expected_perms, dataset, username) + + finally: + if dataset: + dataset.delete() + + def test_user_get_the_edit_permissions_for_the_selected_dataset(self): + """ + Given a user, the compact perms and the resource id, it shoul set the + permissions for the selected resource + """ + try: + expected_perms = { + "view_resourcebase", + "change_dataset_style", + "download_resourcebase", + "change_resourcebase_metadata", + "change_dataset_data", + "change_resourcebase", + } + + dataset, args, username, opts = self._create_arguments(perms_type="edit") + + call_command("set_layers_permissions", *args, **opts) + + self._assert_perms(expected_perms, dataset, username) + finally: + if dataset: + dataset.delete() + + def test_user_get_the_manage_permissions_for_the_selected_dataset(self): + """ + Given a user, the compact perms and the resource id, it shoul set the + permissions for the selected resource + """ + try: + expected_perms = { + "delete_resourcebase", + "change_resourcebase", + "view_resourcebase", + "change_resourcebase_permissions", + "change_dataset_style", + "change_resourcebase_metadata", + "publish_resourcebase", + "change_dataset_data", + "download_resourcebase", + } + + dataset, args, username, opts = self._create_arguments(perms_type="manage") + + call_command("set_layers_permissions", *args, **opts) + + self._assert_perms(expected_perms, dataset, username) + finally: + if dataset: + dataset.delete() + + def test_anonymous_user_cannot_get_edit_permissions(self): + """ + Given the Anonymous user, we should get an error trying to set "edit" permissions. + """ + try: + expected_perms = {} + + dataset, args, username, opts = self._create_arguments(perms_type="edit") + username = "AnonymousUser" + opts["users"] = ["AnonymousUser"] + + call_command("set_layers_permissions", *args, **opts) + + self._assert_perms(expected_perms, dataset, username, assertion=False) + finally: + if dataset: + dataset.delete() + + def test_unset_anonymous_view_permissions(self): + """ + Given the Anonymous user, we should be able to unset any paermission. + """ + try: + expected_perms = {} + + dataset, args, username, opts = self._create_arguments(perms_type="view", mode="unset") + username = "AnonymousUser" + opts["users"] = ["AnonymousUser"] + + call_command("set_layers_permissions", *args, **opts) + + self._assert_perms(expected_perms, dataset, username, assertion=False) + finally: + if dataset: + dataset.delete() + + def _create_arguments(self, perms_type, mode="set"): + dataset = create_single_dataset("dataset_for_management_command") + args = [] + username = get_user_model().objects.exclude(username="admin").exclude(username="AnonymousUser").first().username + opts = { + "permission": perms_type, + "users": [username], + "resources": str(dataset.id), + "delete": True if mode == "unset" else False, + } + + return dataset, args, username, opts + + def _assert_perms(self, expected_perms, dataset, username, assertion=True): + dataset.refresh_from_db() + + perms = dataset.get_all_level_info() + if assertion: + self.assertTrue(username in [user.username for user in perms["users"]]) + actual = set( + itertools.chain.from_iterable( + [perms for user, perms in perms["users"].items() if user.username == username] + ) + ) + self.assertSetEqual(expected_perms, actual) + else: + self.assertFalse(username in [user.username for user in perms["users"]]) + + +class TestDatasetDownloadHandler(GeoNodeBaseTestSupport): + def setUp(self): + user = get_user_model().objects.first() + request = RequestFactory().get("http://test_url.com") + request.user = user + self.dataset = create_single_dataset("test_dataset_for_download") + self.sut = DatasetDownloadHandler(request, self.dataset.alternate) + + def test_download_url_without_original_link(self): + expected_url = reverse("dataset_download", args=[self.dataset.alternate]) + self.assertEqual(expected_url, self.sut.download_url) + + def test_download_url_with_original_link(self): + Link.objects.update_or_create( + resource=self.dataset.resourcebase_ptr, + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) + expected_url = "https://custom_dowonload_url.com" + self.assertEqual(expected_url, self.sut.download_url) + + def test_get_resource_exists(self): + self.assertIsNotNone(self.sut.get_resource()) + + def test_process_dowload(self): + response = self.sut.get_download_response() + self.assertIsNotNone(response) + + +class DummyDownloadHandler(DatasetDownloadHandler): + def get_download_response(self): + return HttpResponse(content=b"abcsfd2") + + +class TestCustomDownloadHandler(GeoNodeBaseTestSupport): + @override_settings(DEFAULT_DATASET_DOWNLOAD_HANDLER="geonode.layers.tests.DummyDownloadHandler") + def test_download_custom_handler(self): + clear_dataset_download_handlers() + dataset = create_single_dataset("test_custom_download_dataset") + url = reverse("dataset_download", args=[dataset.alternate]) + self.client.login(username="admin", password="admin") + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + self.assertEqual(response.content, b"abcsfd2") diff --git a/geonode/layers/translation.py b/geonode/layers/translation.py new file mode 100644 index 0000000..b52e364 --- /dev/null +++ b/geonode/layers/translation.py @@ -0,0 +1,35 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from modeltranslation.translator import translator, TranslationOptions +from geonode.layers.models import Dataset + + +class LayerTranslationOptions(TranslationOptions): + fields = ( + "title", + "abstract", + "purpose", + "constraints_other", + "supplemental_information", + "data_quality_statement", + ) + + +translator.register(Dataset, LayerTranslationOptions) diff --git a/geonode/layers/urls.py b/geonode/layers/urls.py new file mode 100644 index 0000000..883d00d --- /dev/null +++ b/geonode/layers/urls.py @@ -0,0 +1,64 @@ +# ############################################################################## +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ############################################################################## + +from geonode import geoserver # noqa +from geonode.utils import check_ogc_backend +from django.urls import include, re_path + +from . import views + +js_info_dict = { + "packages": ("geonode.layers",), +} + +urlpatterns = [ + # 'geonode.layers.views', + re_path(r"^upload_metadata$", views.dataset_metadata_upload, name="dataset_metadata_upload"), + re_path(r"^load_dataset_data$", views.load_dataset_data, name="load_dataset_data"), + re_path(r"^(?P[^/]*)/metadata$", views.dataset_metadata, name="dataset_metadata"), + re_path( + r"^(?P[^/]*)/metadata_advanced$", views.dataset_metadata_advanced, name="dataset_metadata_advanced" + ), + re_path( + r"^(?P[^/]*)/(?P[^/]*)/granule_remove$", + views.dataset_granule_remove, + name="dataset_granule_remove", + ), + re_path(r"^(?P[^/]*)/get$", views.get_dataset, name="get_dataset"), + re_path(r"^(?P[^/]*)/metadata_detail$", views.dataset_metadata_detail, name="dataset_metadata_detail"), + re_path(r"^(?P[^/]*)/metadata_upload$", views.dataset_metadata_upload, name="dataset_metadata_upload"), + re_path(r"^(?P[^/]+)/embed$", views.dataset_embed, name="dataset_embed"), + re_path(r"^(?P[^/]*)/style_upload$", views.dataset_sld_upload, name="dataset_sld_upload"), + re_path( + r"^(?P[^/]*)/feature_catalogue$", views.dataset_feature_catalogue, name="dataset_feature_catalogue" + ), + re_path(r"^metadata/batch/$", views.dataset_batch_metadata, name="dataset_batch_metadata"), + re_path(r"^(?P[^/]*)/dataset_download$", views.dataset_download, name="dataset_download"), + re_path(r"^", include("geonode.layers.api.urls")), +] + +# -- Deprecated url routes for Geoserver authentication -- remove after GeoNode 2.1 +# -- Use /gs/acls, gs/resolve_user/, gs/download instead +if check_ogc_backend(geoserver.BACKEND_PACKAGE): + from geonode.geoserver.views import dataset_acls, resolve_user + + urlpatterns = [ # 'geonode.geoserver.views', + re_path(r"^acls/?$", dataset_acls, name="dataset_acls_dep"), + re_path(r"^resolve_user/?$", resolve_user, name="dataset_resolve_user_dep"), + ] + urlpatterns diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py new file mode 100644 index 0000000..b710211 --- /dev/null +++ b/geonode/layers/utils.py @@ -0,0 +1,537 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +"""Utilities for managing GeoNode layers +""" + +# Standard Modules +import copy +import re +import os +import glob +import string +import logging +import tarfile + +from osgeo import gdal, osr +from zipfile import ZipFile, is_zipfile +from random import choice + +# Django functionality +from django.conf import settings +from django.db.models import Q +from django.contrib.auth.models import Group +from django.contrib.auth import get_user_model +from django.utils.module_loading import import_string +from django.core.exceptions import ObjectDoesNotExist + +from geonode.security.permissions import PermSpec, PermSpecCompact +from geonode.storage.manager import storage_manager + +# Geonode functionality +from geonode.base.models import Region +from geonode.utils import check_ogc_backend +from geonode import GeoNodeException, geoserver +from geonode.layers.models import shp_exts, csv_exts, vec_exts, cov_exts, Dataset + +READ_PERMISSIONS = ["view_resourcebase"] +WRITE_PERMISSIONS = ["change_dataset_data", "change_dataset_style", "change_resourcebase_metadata"] +DOWNLOAD_PERMISSIONS = ["download_resourcebase"] +OWNER_PERMISSIONS = [ + "change_resourcebase", + "delete_resourcebase", + "change_resourcebase_permissions", + "publish_resourcebase", +] + +logger = logging.getLogger("geonode.layers.utils") + +_separator = f"\n{'-' * 100}\n" + + +def _clean_string(str, regex=r"(^[^a-zA-Z\._]+)|([^a-zA-Z\._0-9]+)", replace="_"): + """ + Replaces a string that matches the regex with the replacement. + """ + regex = re.compile(regex) + + if str[0].isdigit(): + str = replace + str + + return regex.sub(replace, str) + + +def resolve_regions(regions): + regions_resolved = [] + regions_unresolved = [] + if regions and len(regions) > 0: + for region in regions: + try: + if region.isnumeric(): + region_resolved = Region.objects.get(id=int(region)) + else: + region_resolved = Region.objects.get(Q(name__iexact=region) | Q(code__iexact=region)) + regions_resolved.append(region_resolved) + except ObjectDoesNotExist: + regions_unresolved.append(region) + + return regions_resolved, regions_unresolved + + +def get_files(filename): + """Converts the data to Shapefiles or Geotiffs and returns + a dictionary with all the required files + """ + files = {} + + # Verify if the filename is in ascii format. + try: + filename.encode("ascii") + except UnicodeEncodeError: + msg = f"Please use only characters from the english alphabet for the filename. '{os.path.basename(filename).encode('UTF-8', 'strict')}' is not yet supported." + raise GeoNodeException(msg) + + # Let's unzip the filname in case it is a ZIP file + from geonode.utils import unzip_file, mkdtemp + + tempdir = None + if is_zipfile(filename): + tempdir = mkdtemp() + _filename = unzip_file(filename, ".shp", tempdir=tempdir) + if not _filename: + # We need to iterate files as filename could be the zipfile + import ntpath + from geonode.upload.utils import _SUPPORTED_EXT + + file_basename, file_ext = ntpath.splitext(filename) + for item in os.listdir(tempdir): + item_basename, item_ext = ntpath.splitext(item) + if ntpath.basename(item_basename) == ntpath.basename(file_basename) and ( + item_ext.lower() in _SUPPORTED_EXT + ): + filename = os.path.join(tempdir, item) + break + else: + filename = _filename + + # Make sure the file exists. + if not os.path.exists(filename): + msg = f"Could not open {filename}. Make sure you are using a valid file" + logger.debug(msg) + raise GeoNodeException(msg) + + base_name, extension = os.path.splitext(filename) + # Replace special characters in filenames - []{}() + glob_name = re.sub(r"([\[\]\(\)\{\}])", r"[\g<1>]", base_name) + + if extension.lower() == ".shp": + required_extensions = dict(shp=".[sS][hH][pP]", dbf=".[dD][bB][fF]", shx=".[sS][hH][xX]") + for ext, pattern in required_extensions.items(): + matches = glob.glob(glob_name + pattern) + if len(matches) == 0: + msg = ( + f"Expected helper file {base_name}.{ext} does not exist; a Shapefile " + "requires helper files with the following extensions: " + f"{list(required_extensions.keys())}" + ) + raise GeoNodeException(msg) + elif len(matches) > 1: + msg = ( + "Multiple helper files for %s exist; they need to be " "distinct by spelling and not just case." + ) % filename + raise GeoNodeException(msg) + else: + files[ext] = matches[0] + + matches = glob.glob(f"{glob_name}.[pP][rR][jJ]") + if len(matches) == 1: + files["prj"] = matches[0] + elif len(matches) > 1: + msg = ( + "Multiple helper files for %s exist; they need to be " "distinct by spelling and not just case." + ) % filename + raise GeoNodeException(msg) + + elif extension.lower() in cov_exts: + files[extension.lower().replace(".", "")] = filename + + # Only for GeoServer + if check_ogc_backend(geoserver.BACKEND_PACKAGE): + matches = glob.glob(f"{os.path.dirname(glob_name)}.[sS][lL][dD]") + if len(matches) == 1: + files["sld"] = matches[0] + else: + matches = glob.glob(f"{glob_name}.[sS][lL][dD]") + if len(matches) == 1: + files["sld"] = matches[0] + elif len(matches) > 1: + msg = ( + "Multiple style files (sld) for %s exist; they need to be " + "distinct by spelling and not just case." + ) % filename + raise GeoNodeException(msg) + + matches = glob.glob(f"{glob_name}.[xX][mM][lL]") + + # shapefile XML metadata is sometimes named base_name.shp.xml + # try looking for filename.xml if base_name.xml does not exist + if len(matches) == 0: + matches = glob.glob(f"{filename}.[xX][mM][lL]") + + if len(matches) == 1: + files["xml"] = matches[0] + elif len(matches) > 1: + msg = ("Multiple XML files for %s exist; they need to be " "distinct by spelling and not just case.") % filename + raise GeoNodeException(msg) + + return files, tempdir + + +def dataset_type(filename): + """Finds out if a filename is a Feature or a Vector + returns a gsconfig resource_type string + that can be either 'featureType' or 'coverage' + """ + base_name, extension = os.path.splitext(filename) + + if extension.lower() == ".zip": + zf = ZipFile(filename, allowZip64=True) + # ZipFile doesn't support with statement in 2.6, so don't do it + with zf: + for n in zf.namelist(): + b, e = os.path.splitext(n.lower()) + if e in shp_exts or e in cov_exts or e in csv_exts: + extension = e + + if extension.lower() == ".tar" or filename.endswith(".tar.gz"): + tf = tarfile.open(filename) + # TarFile doesn't support with statement in 2.6, so don't do it + with tf: + for n in tf.getnames(): + b, e = os.path.splitext(n.lower()) + if e in shp_exts or e in cov_exts or e in csv_exts: + extension = e + + if extension.lower() in vec_exts: + return "vector" + elif extension.lower() in cov_exts: + return "raster" + else: + msg = f"Saving of extension [{extension}] is not implemented" + raise GeoNodeException(msg) + + +def get_valid_name(dataset_name): + """ + Create a brand new name + """ + name = _clean_string(dataset_name) + proposed_name = name + while Dataset.objects.filter(name=proposed_name).exists(): + possible_chars = string.ascii_lowercase + string.digits + suffix = "".join([choice(possible_chars) for i in range(4)]) + proposed_name = f"{name}_{suffix}" + logger.debug("Requested name already used; adjusting name " f"[{dataset_name}] => [{proposed_name}]") + + return proposed_name + + +def get_valid_dataset_name(layer, overwrite): + """Checks if the layer is a string and fetches it from the database.""" + # The first thing we do is get the layer name string + if isinstance(layer, Dataset): + dataset_name = layer.name + elif isinstance(layer, str): + dataset_name = str(layer) + else: + msg = "You must pass either a filename or a GeoNode dataset object" + raise GeoNodeException(msg) + + if overwrite: + return dataset_name + else: + return get_valid_name(dataset_name) + + +def get_default_user(): + """Create a default user""" + superusers = get_user_model().objects.filter(is_superuser=True).order_by("id") + if superusers.exists(): + # Return the first created superuser + return superusers[0] + else: + raise GeoNodeException( + "You must have an admin account configured " + "before importing data. " + "Try: django-admin.py createsuperuser" + ) + + +def is_vector(filename): + __, extension = os.path.splitext(filename) + + if extension in vec_exts: + return True + else: + return False + + +def is_raster(filename): + __, extension = os.path.splitext(filename) + + if extension in cov_exts: + return True + else: + return False + + +def get_resolution(filename): + try: + gtif = gdal.Open(filename) + gt = gtif.GetGeoTransform() + __, resx, __, __, __, resy = gt + resolution = f"{resx} {resy}" + return resolution + except Exception: + return None + + +def get_bbox(filename): + """Return bbox in the format [xmin,xmax,ymin,ymax].""" + from django.contrib.gis.gdal import DataSource, SRSException + + srid = 4326 + bbox_x0, bbox_y0, bbox_x1, bbox_y1 = -180, -90, 180, 90 + + try: + if is_vector(filename): + y_min = -90 + y_max = 90 + x_min = -180 + x_max = 180 + datasource = DataSource(filename) + layer = datasource[0] + bbox_x0, bbox_y0, bbox_x1, bbox_y1 = layer.extent.tuple + srs = layer.srs + try: + if not srs: + raise GeoNodeException("Invalid Projection. Dataset is missing CRS!") + srs.identify_epsg() + except SRSException: + pass + epsg_code = srs.srid + # can't find epsg code, then check if bbox is within the 4326 boundary + if epsg_code is None and ( + x_min <= bbox_x0 <= x_max + and x_min <= bbox_x1 <= x_max + and y_min <= bbox_y0 <= y_max + and y_min <= bbox_y1 <= y_max + ): + # set default epsg code + epsg_code = "4326" + elif epsg_code is None: + # otherwise, stop the upload process + raise GeoNodeException("Invalid Datasets. " "Needs an authoritative SRID in its CRS to be accepted") + + # eliminate default EPSG srid as it will be added when this function returned + srid = epsg_code if epsg_code else "4326" + elif is_raster(filename): + gtif = gdal.Open(filename) + gt = gtif.GetGeoTransform() + prj = gtif.GetProjection() + srs = osr.SpatialReference(wkt=prj) + cols = gtif.RasterXSize + rows = gtif.RasterYSize + + ext = [] + xarr = [0, cols] + yarr = [0, rows] + + # Get the extent. + for px in xarr: + for py in yarr: + x = gt[0] + (px * gt[1]) + (py * gt[2]) + y = gt[3] + (px * gt[4]) + (py * gt[5]) + ext.append([x, y]) + + yarr.reverse() + + # ext has four corner points, get a bbox from them. + # order is important, so make sure min and max is correct. + bbox_x0 = min(ext[0][0], ext[2][0]) + bbox_y0 = min(ext[0][1], ext[2][1]) + bbox_x1 = max(ext[0][0], ext[2][0]) + bbox_y1 = max(ext[0][1], ext[2][1]) + srid = srs.GetAuthorityCode(None) if srs else "4326" + except Exception: + pass + + return [bbox_x0, bbox_x1, bbox_y0, bbox_y1, f"EPSG:{str(srid)}"] + + +def delete_orphaned_datasets(): + """Delete orphaned layer files.""" + deleted = [] + _, files = storage_manager.listdir("layers") + + for filename in files: + if Dataset.objects.filter(file__icontains=filename).count() == 0: + logger.debug(f"Deleting orphaned dataset file {filename}") + try: + storage_manager.delete(os.path.join("layers", filename)) + deleted.append(filename) + except NotImplementedError as e: + logger.error(f"Failed to delete orphaned dataset file '{filename}': {e}") + + return deleted + + +def surrogate_escape_string(input_string, source_character_set): + """ + Escapes a given input string using the provided source character set, + using the `surrogateescape` codec error handler. + """ + + return input_string.encode(source_character_set, "surrogateescape").decode("utf-8", "surrogateescape") + + +def set_datasets_permissions( + permissions_name, resources_names=None, users_usernames=None, groups_names=None, delete_flag=False, verbose=False +): + # here to avoid circular import + from geonode.resource.manager import resource_manager + + # Processing information + resources_as_pk = [] + for el in resources_names or []: + if isinstance(el, str) and not el.isnumeric(): + res = Dataset.objects.filter(Q(title=el) | Q(name=el)) + if res.exists(): + resources_as_pk.append(res.first().pk) + else: + resources_as_pk.append(el) + + not_found = [] + final_perms_payload = {} + + for rpk in resources_as_pk: + resource = Dataset.objects.filter(pk=rpk) + if not resource.exists(): + not_found.append(rpk) + logger.error(f"Resource named: {rpk} not found, skipping....") + continue + else: + # creating the payload from the CompactPermissions like we do in the UI. + # the result will be a payload with the compact permissions list required + # for the selected resource + resource = resource.first() + # getting the actual permissions available for the dataset + original_perms = PermSpec(resource.get_all_level_info(), resource) + new_perms_payload = {"organizations": [], "users": [], "groups": []} + # if the username is specified, we add them to the payload with the compact + # perm value + if users_usernames: + User = get_user_model() + for _user in users_usernames: + try: + new_perms_payload["users"].append( + {"id": User.objects.get(username=_user).pk, "permissions": permissions_name} + ) + except User.DoesNotExist: + logger.warning(f"The user {_user} does not exists. " "It has been skipped.") + # GROUPS + # if the group is specified, we add them to the payload with the compact + # perm value + if groups_names: + for group_name in groups_names: + try: + new_perms_payload["groups"].append( + {"id": Group.objects.get(name=group_name).pk, "permissions": permissions_name} + ) + except Group.DoesNotExist: + logger.warning(f"The group {group_name} does not exists. " "It has been skipped.") + # Using the compact permissions payload to calculate the permissions + # that we want to give for each user/group + # This part is in common with the permissions API + new_compact_perms = PermSpecCompact(new_perms_payload, resource) + copy_compact_perms = copy.deepcopy(new_compact_perms) + + perms_spec_compact_resource = PermSpecCompact(original_perms.compact, resource) + perms_spec_compact_resource.merge(new_compact_perms) + + final_perms_payload = perms_spec_compact_resource.extended + # if the delete flag is set, we must delete the permissions for the input user/group + if delete_flag: + # since is a delete operation, we must remove the users/group from the resource + # so this will return the updated dict without the user/groups to be removed + final_perms_payload["users"] = { + _user: _perms + for _user, _perms in perms_spec_compact_resource.extended["users"].items() + if _user not in copy_compact_perms.extended["users"] + } + final_perms_payload["groups"] = { + _group: _perms + for _group, _perms in perms_spec_compact_resource.extended["groups"].items() + if _group not in copy_compact_perms.extended["groups"] + } + if final_perms_payload["users"].get("AnonymousUser") is None and final_perms_payload["groups"].get( + "anonymous" + ): + final_perms_payload["groups"].pop("anonymous") + + # calling the resource manager to set the permissions + resource_manager.set_permissions(resource.uuid, instance=resource, permissions=final_perms_payload) + + +def get_uuid_handler(): + return import_string(settings.LAYER_UUID_HANDLER) + + +default_dataset_download_handler = None +dataset_download_handler_list = [] + + +def get_download_handlers(): + if not dataset_download_handler_list and getattr(settings, "DATASET_DOWNLOAD_HANDLERS", None): + dataset_download_handler_list.append(import_string(settings.DATASET_DOWNLOAD_HANDLERS[0])) + + return dataset_download_handler_list + + +def get_default_dataset_download_handler(): + global default_dataset_download_handler + if not default_dataset_download_handler and getattr(settings, "DEFAULT_DATASET_DOWNLOAD_HANDLER", None): + default_dataset_download_handler = import_string(settings.DEFAULT_DATASET_DOWNLOAD_HANDLER) + + return default_dataset_download_handler + + +def set_default_dataset_download_handler(handler): + global default_dataset_download_handler + handler_module = import_string(handler) + if handler_module not in dataset_download_handler_list: + dataset_download_handler_list.append(handler_module) + + default_dataset_download_handler = handler_module + + +def clear_dataset_download_handlers(): + global default_dataset_download_handler + dataset_download_handler_list.clear() + default_dataset_download_handler = None diff --git a/geonode/layers/views.py b/geonode/layers/views.py new file mode 100644 index 0000000..7760852 --- /dev/null +++ b/geonode/layers/views.py @@ -0,0 +1,755 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import re +import os +import json +import decimal +import logging +import warnings +import traceback + +from owslib.wfs import WebFeatureService + +from django.conf import settings + +from django.db.models import F +from django.http import Http404 +from django.contrib import messages +from django.shortcuts import render +from django.contrib.auth import get_user_model +from django.utils.translation import gettext_lazy as _ +from django.core.exceptions import PermissionDenied +from django.forms.models import inlineformset_factory +from django.template.response import TemplateResponse +from django.contrib.auth.decorators import login_required +from django.views.decorators.csrf import csrf_exempt +from django.http import HttpResponse, HttpResponseRedirect +from django.views.decorators.clickjacking import xframe_options_exempt + +from geonode import geoserver +from geonode.resource.manager import resource_manager +from geonode.base.auth import get_or_create_token +from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm +from geonode.base.views import batch_modify +from geonode.base.models import Thesaurus, TopicCategory +from geonode.decorators import check_keyword_write_perms +from geonode.layers.forms import DatasetForm, DatasetTimeSerieForm, LayerAttributeForm +from geonode.layers.models import Dataset, Attribute +from geonode.layers.utils import ( + get_default_dataset_download_handler, +) +from geonode.services.models import Service +from geonode.base import register_event +from geonode.monitoring.models import EventType +from geonode.groups.models import GroupProfile +from geonode.security.utils import get_user_visible_groups +from geonode.people.forms import ProfileForm +from geonode.utils import check_ogc_backend, llbbox_to_mercator, resolve_object +from geonode.geoserver.helpers import ogc_server_settings + +if check_ogc_backend(geoserver.BACKEND_PACKAGE): + from geonode.geoserver.helpers import gs_catalog + +CONTEXT_LOG_FILE = ogc_server_settings.LOG_FILE + +logger = logging.getLogger("geonode.layers.views") + +DEFAULT_SEARCH_BATCH_SIZE = 10 +MAX_SEARCH_BATCH_SIZE = 25 +GENERIC_UPLOAD_ERROR = _( + "There was an error while attempting to upload your data. \ +Please try again, or contact and administrator if the problem continues." +) + +METADATA_UPLOADED_PRESERVE_ERROR = _( + "Note: this dataset's orginal metadata was \ +populated and preserved by importing a metadata XML file. This metadata cannot be edited." +) + +_PERMISSION_MSG_DELETE = _("You are not permitted to delete this dataset") +_PERMISSION_MSG_GENERIC = _("You do not have permissions for this dataset.") +_PERMISSION_MSG_MODIFY = _("You are not permitted to modify this dataset") +_PERMISSION_MSG_METADATA = _("You are not permitted to modify this dataset's metadata") +_PERMISSION_MSG_VIEW = _("You are not permitted to view this dataset") + + +def log_snippet(log_file): + if not log_file or not os.path.isfile(log_file): + return f"No log file at {log_file}" + + with open(log_file) as f: + f.seek(0, 2) # Seek @ EOF + fsize = f.tell() # Get Size + f.seek(max(fsize - 10024, 0), 0) # Set pos @ last n chars + return f.read() + + +def _resolve_dataset(request, alternate, permission="base.view_resourcebase", msg=_PERMISSION_MSG_GENERIC, **kwargs): + """ + Resolve the layer by the provided typename (which may include service name) and check the optional permission. + """ + service_typename = alternate.split(":", 1) + for _s in Service.objects.filter(name__startswith=service_typename[0]): + query = { + "alternate": service_typename[1], + "remote_service": _s, + } + try: + return resolve_object(request, Dataset, query, permission=permission, permission_msg=msg, **kwargs) + except (Dataset.DoesNotExist, Http404) as e: + logger.debug(e) + if len(service_typename) > 1 and ":" in service_typename[1]: + if service_typename[0]: + query = {"store": service_typename[0], "alternate": service_typename[1]} + else: + query = {"alternate": service_typename[1]} + else: + query = {"alternate": alternate} + test_query = Dataset.objects.filter(**query) + if test_query.count() > 1 and test_query.exclude(subtype="remote").count() == 1: + query = {"id": test_query.exclude(subtype="remote").last().id} + elif test_query.count() > 1: + query = {"id": test_query.last().id} + return resolve_object(request, Dataset, query, permission=permission, permission_msg=msg, **kwargs) + + +# Loads the data using the OWS lib when the "Do you want to filter it" +# button is clicked. +def load_dataset_data(request): + context_dict = {} + data_dict = json.loads(request.POST.get("json_data")) + layername = data_dict["dataset_name"] + filtered_attributes = "" + if not isinstance(data_dict["filtered_attributes"], str): + filtered_attributes = [x for x in data_dict["filtered_attributes"] if "/load_dataset_data" not in x] + name = layername if ":" not in layername else layername.split(":")[1] + location = f"{(settings.OGC_SERVER['default']['LOCATION'])}wms" + headers = {} + if request and "access_token" in request.session: + access_token = request.session["access_token"] + headers["Authorization"] = f"Bearer {access_token}" + + try: + wfs = WebFeatureService(location, version="1.1.0", headers=headers) + response = wfs.getfeature(typename=name, propertyname=filtered_attributes, outputFormat="application/json") + x = response.read() + x = json.loads(x) + features_response = json.dumps(x) + decoded = json.loads(features_response) + decoded_features = decoded["features"] + properties = {} + for key in decoded_features[0]["properties"]: + properties[key] = [] + + # loop the dictionary based on the values on the list and add the properties + # in the dictionary (if doesn't exist) together with the value + from collections.abc import Iterable + + for i in range(len(decoded_features)): + for key, value in decoded_features[i]["properties"].items(): + if ( + value != "" + and isinstance(value, (str, int, float)) + and ((isinstance(value, Iterable) and "/load_dataset_data" not in value) or value) + ): + properties[key].append(value) + + for key in properties: + properties[key] = list(set(properties[key])) + properties[key].sort() + + context_dict["feature_properties"] = properties + except Exception: + traceback.print_exc() + logger.error("Possible error with OWSLib.") + return HttpResponse(json.dumps(context_dict), content_type="application/json") + + +def dataset_feature_catalogue(request, layername, template="../../catalogue/templates/catalogue/feature_catalogue.xml"): + try: + layer = _resolve_dataset(request, layername) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not layer: + raise Http404(_("Not found")) + + if layer.subtype != "vector": + out = {"success": False, "errors": "layer is not a feature type"} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + + attributes = [] + + for attrset in layer.attribute_set.order_by("display_order"): + attr = {"name": attrset.attribute, "type": attrset.attribute_type} + attributes.append(attr) + + context_dict = { + "dataset": layer, + "attributes": attributes, + "metadata": settings.PYCSW["CONFIGURATION"]["metadata:main"], + } + register_event(request, "view", layer) + return render(request, template, context=context_dict, content_type="application/xml") + + +@login_required +@check_keyword_write_perms +def dataset_metadata( + request, + layername, + template="datasets/dataset_metadata.html", + panel_template="layouts/panels.html", + custom_metadata=None, + ajax=True, +): + try: + layer = _resolve_dataset(request, layername, "base.change_resourcebase_metadata", _PERMISSION_MSG_METADATA) + except PermissionDenied as e: + return HttpResponse(Exception(_("Not allowed"), e), status=403) + except Exception as e: + raise Http404(Exception(_("Not found"), e)) + if not layer: + raise Http404(_("Not found")) + dataset_attribute_set = inlineformset_factory( + Dataset, + Attribute, + extra=0, + form=LayerAttributeForm, + ) + current_keywords = [keyword.name for keyword in layer.keywords.all()] + topic_category = layer.category + + topic_thesaurus = layer.tkeywords.all() + + # Add metadata_author or poc if missing + layer.add_missing_metadata_author_or_poc() + + # assert False, str(dataset_bbox) + config = layer.attribute_config() + + # Add required parameters for GXP lazy-loading + dataset_bbox = layer.bbox + bbox = [float(coord) for coord in list(dataset_bbox[0:4])] + if hasattr(layer, "srid"): + config["crs"] = {"type": "name", "properties": layer.srid} + config["srs"] = getattr(settings, "DEFAULT_MAP_CRS", "EPSG:3857") + config["bbox"] = bbox if config["srs"] != "EPSG:3857" else llbbox_to_mercator([float(coord) for coord in bbox]) + config["title"] = layer.title + config["queryable"] = True + + # Update count for popularity ranking, + # but do not includes admins or resource owners + if request.user != layer.owner and not request.user.is_superuser: + Dataset.objects.filter(id=layer.id).update(popular_count=F("popular_count") + 1) + + if request.method == "POST": + if layer.metadata_uploaded_preserve: # layer metadata cannot be edited + out = {"success": False, "errors": METADATA_UPLOADED_PRESERVE_ERROR} + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + + thumbnail_url = layer.thumbnail_url + dataset_form = DatasetForm(request.POST, instance=layer, prefix="resource", user=request.user) + + if not dataset_form.is_valid(): + logger.error(f"Dataset Metadata form is not valid: {dataset_form.errors}") + out = { + "success": False, + "errors": [f"{x}: {y[0].messages[0]}" for x, y in dataset_form.errors.as_data().items()], + } + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + if not layer.thumbnail_url: + layer.thumbnail_url = thumbnail_url + attribute_form = dataset_attribute_set( + request.POST, + instance=layer, + prefix="dataset_attribute_set", + queryset=Attribute.objects.order_by("display_order"), + ) + if not attribute_form.is_valid(): + logger.error(f"Dataset Attributes form is not valid: {attribute_form.errors}") + out = { + "success": False, + "errors": [re.sub(re.compile("<.*?>"), "", str(err)) for err in attribute_form.errors], + } + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + category_form = CategoryForm( + request.POST, + prefix="category_choice_field", + initial=( + int(request.POST["category_choice_field"]) + if "category_choice_field" in request.POST and request.POST["category_choice_field"] + else None + ), + ) + if not category_form.is_valid(): + logger.error(f"Dataset Category form is not valid: {category_form.errors}") + out = { + "success": False, + "errors": [re.sub(re.compile("<.*?>"), "", str(err)) for err in category_form.errors], + } + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + if hasattr(settings, "THESAURUS"): + tkeywords_form = TKeywordForm(request.POST) + else: + tkeywords_form = ThesaurusAvailableForm(request.POST, prefix="tkeywords") + # set initial values for thesaurus form + if not tkeywords_form.is_valid(): + logger.error(f"Dataset Thesauri Keywords form is not valid: {tkeywords_form.errors}") + out = { + "success": False, + "errors": [re.sub(re.compile("<.*?>"), "", str(err)) for err in tkeywords_form.errors], + } + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + + timeseries_form = DatasetTimeSerieForm(request.POST, instance=layer, prefix="timeseries") + if not timeseries_form.is_valid(): + out = { + "success": False, + "errors": [f"{x}: {y[0].messages[0]}" for x, y in timeseries_form.errors.as_data().items()], + } + logger.error(f"{out.get('errors')}") + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + elif ( + layer.has_time + and timeseries_form.is_valid() + and not timeseries_form.cleaned_data.get("attribute", "") + and not timeseries_form.cleaned_data.get("end_attribute", "") + ): + out = { + "success": False, + "errors": [ + "The Timeseries configuration is invalid. Please select at least one option between the `attribute` and `end_attribute`, otherwise remove the 'has_time' flag" + ], + } + logger.error(f"{out.get('errors')}") + return HttpResponse(json.dumps(out), content_type="application/json", status=400) + else: + dataset_form = DatasetForm(instance=layer, prefix="resource", user=request.user) + dataset_form.disable_keywords_widget_for_non_superuser(request.user) + attribute_form = dataset_attribute_set( + instance=layer, prefix="dataset_attribute_set", queryset=Attribute.objects.order_by("display_order") + ) + category_form = CategoryForm( + prefix="category_choice_field", initial=topic_category.id if topic_category else None + ) + + gs_layer = gs_catalog.get_layer(name=layer.name) + initial = {} + if gs_layer is not None and layer.has_time: + gs_time_info = gs_layer.resource.metadata.get("time") + if gs_time_info.enabled: + _attr = layer.attributes.filter(attribute=gs_time_info.attribute).first() + initial["attribute"] = _attr.pk if _attr else None + if gs_time_info.end_attribute is not None: + end_attr = layer.attributes.filter(attribute=gs_time_info.end_attribute).first() + initial["end_attribute"] = end_attr.pk if end_attr else None + initial["presentation"] = gs_time_info.presentation + lookup_value = sorted(list(gs_time_info._lookup), key=lambda x: x[1], reverse=True) + if gs_time_info.resolution is not None: + res = gs_time_info.resolution // 1000 + for el in lookup_value: + if res % el[1] == 0: + initial["precision_value"] = res // el[1] + initial["precision_step"] = el[0] + break + else: + initial["precision_value"] = gs_time_info.resolution + initial["precision_step"] = "seconds" + + timeseries_form = DatasetTimeSerieForm(instance=layer, prefix="timeseries", initial=initial) + + # Create THESAURUS widgets + lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, "THESAURUS_DEFAULT_LANG") else "en" + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + warnings.warn( + "The settings for Thesaurus has been moved to Model, \ + this feature will be removed in next releases", + DeprecationWarning, + ) + dataset_tkeywords = layer.tkeywords.all() + tkeywords_list = "" + if dataset_tkeywords and len(dataset_tkeywords) > 0: + tkeywords_ids = dataset_tkeywords.values_list("id", flat=True) + if hasattr(settings, "THESAURUS") and settings.THESAURUS: + el = settings.THESAURUS + thesaurus_name = el["name"] + try: + t = Thesaurus.objects.get(identifier=thesaurus_name) + for tk in t.thesaurus.filter(pk__in=tkeywords_ids): + tkl = tk.keyword.filter(lang=lang) + if len(tkl) > 0: + tkl_ids = ",".join(map(str, tkl.values_list("id", flat=True))) + tkeywords_list += f",{tkl_ids}" if len(tkeywords_list) > 0 else tkl_ids + except Exception: + tb = traceback.format_exc() + logger.error(tb) + tkeywords_form = TKeywordForm(instance=layer) + else: + tkeywords_form = ThesaurusAvailableForm(prefix="tkeywords") + # set initial values for thesaurus form + for tid in tkeywords_form.fields: + values = [] + values = [keyword.id for keyword in topic_thesaurus if int(tid) == keyword.thesaurus.id] + tkeywords_form.fields[tid].initial = values + if ( + request.method == "POST" + and dataset_form.is_valid() + and attribute_form.is_valid() + and category_form.is_valid() + and tkeywords_form.is_valid() + and timeseries_form.is_valid() + ): + new_category = None + if ( + category_form + and "category_choice_field" in category_form.cleaned_data + and category_form.cleaned_data["category_choice_field"] + ): + new_category = TopicCategory.objects.get(id=int(category_form.cleaned_data["category_choice_field"])) + + for form in attribute_form.cleaned_data: + la = Attribute.objects.get(id=int(form["id"].id)) + la.description = form["description"] + la.attribute_label = form["attribute_label"] + la.visible = form["visible"] + la.display_order = form["display_order"] + la.featureinfo_type = form["featureinfo_type"] + la.save() + + # update contact roles + layer.set_contact_roles_from_metadata_edit(dataset_form) + layer.save() + + new_keywords = current_keywords if request.keyword_readonly else dataset_form.cleaned_data["keywords"] + new_regions = [x.strip() for x in dataset_form.cleaned_data["regions"]] + + layer.keywords.clear() + if new_keywords: + layer.keywords.add(*new_keywords) + layer.regions.clear() + if new_regions: + layer.regions.add(*new_regions) + layer.category = new_category + + from geonode.upload.models import Upload + + up_sessions = Upload.objects.filter(resource_id=layer.resourcebase_ptr_id) + if up_sessions.exists() and up_sessions[0].user != layer.owner: + up_sessions.update(user=layer.owner) + + dataset_form.save_linked_resources() + + register_event(request, EventType.EVENT_CHANGE_METADATA, layer) + if not ajax: + return HttpResponseRedirect(layer.get_absolute_url()) + + message = layer.alternate + + try: + if not tkeywords_form.is_valid(): + return HttpResponse(json.dumps({"message": "Invalid thesaurus keywords"}, status_code=400)) + + thesaurus_setting = getattr(settings, "THESAURUS", None) + if thesaurus_setting: + tkeywords_data = tkeywords_form.cleaned_data["tkeywords"] + tkeywords_data = tkeywords_data.filter(thesaurus__identifier=thesaurus_setting["name"]) + layer.tkeywords.set(tkeywords_data) + elif Thesaurus.objects.all().exists(): + fields = tkeywords_form.cleaned_data + layer.tkeywords.set(tkeywords_form.cleanx(fields)) + + except Exception: + tb = traceback.format_exc() + logger.error(tb) + + vals = {} + if "group" in dataset_form.changed_data: + vals["group"] = dataset_form.cleaned_data.get("group") + if any([x in dataset_form.changed_data for x in ["is_approved", "is_published"]]): + vals["is_approved"] = dataset_form.cleaned_data.get("is_approved", layer.is_approved) + vals["is_published"] = dataset_form.cleaned_data.get("is_published", layer.is_published) + + layer.has_time = dataset_form.cleaned_data.get("has_time", layer.has_time) + + if ( + layer.is_vector() + and timeseries_form.cleaned_data + and ("has_time" in dataset_form.changed_data or timeseries_form.changed_data) + ): + ts = timeseries_form.cleaned_data + end_attr = layer.attributes.get(pk=ts.get("end_attribute")).attribute if ts.get("end_attribute") else None + start_attr = layer.attributes.get(pk=ts.get("attribute")).attribute if ts.get("attribute") else None + resource_manager.exec( + "set_time_info", + None, + instance=layer, + time_info={ + "attribute": start_attr, + "end_attribute": end_attr, + "presentation": ts.get("presentation", None), + "precision_value": ts.get("precision_value", None), + "precision_step": ts.get("precision_step", None), + "enabled": dataset_form.cleaned_data.get("has_time", False), + }, + ) + + resource_manager.update( + layer.uuid, + instance=layer, + notify=True, + vals=vals, + extra_metadata=json.loads(dataset_form.cleaned_data["extra_metadata"]), + ) + + return HttpResponse(json.dumps({"message": message})) + + if not request.user.can_publish(layer): + dataset_form.fields["is_published"].widget.attrs.update({"disabled": "true"}) + if not request.user.can_approve(layer): + dataset_form.fields["is_approved"].widget.attrs.update({"disabled": "true"}) + + # define contact role forms + contact_role_forms_context = {} + for role in layer.get_multivalue_role_property_names(): + dataset_form.fields[role].initial = [p.username for p in layer.__getattribute__(role)] + role_form = ProfileForm(prefix=role) + role_form.hidden = True + contact_role_forms_context[f"{role}_form"] = role_form + + metadata_author_groups = get_user_visible_groups(request.user) + + register_event(request, "view_metadata", layer) + return render( + request, + template, + context={ + "resource": layer, + "dataset": layer, + "panel_template": panel_template, + "custom_metadata": custom_metadata, + "dataset_form": dataset_form, + "attribute_form": attribute_form, + "timeseries_form": timeseries_form, + "category_form": category_form, + "tkeywords_form": tkeywords_form, + "preview": getattr(settings, "GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore"), + "crs": getattr(settings, "DEFAULT_MAP_CRS", "EPSG:3857"), + "metadataxsl": getattr(settings, "GEONODE_CATALOGUE_METADATA_XSL", True), + "freetext_readonly": getattr(settings, "FREETEXT_KEYWORDS_READONLY", False), + "metadata_author_groups": metadata_author_groups, + "TOPICCATEGORY_MANDATORY": getattr(settings, "TOPICCATEGORY_MANDATORY", False), + "GROUP_MANDATORY_RESOURCES": getattr(settings, "GROUP_MANDATORY_RESOURCES", False), + "UI_MANDATORY_FIELDS": list( + set(getattr(settings, "UI_DEFAULT_MANDATORY_FIELDS", [])) + | set(getattr(settings, "UI_REQUIRED_FIELDS", [])) + ), + **contact_role_forms_context, + "UI_ROLES_IN_TOGGLE_VIEW": layer.get_ui_toggled_role_property_names(), + }, + ) + + +@login_required +def dataset_metadata_advanced(request, layername): + return dataset_metadata(request, layername, template="datasets/dataset_metadata_advanced.html") + + +@csrf_exempt +def dataset_download(request, layername): + handler = get_default_dataset_download_handler() + return handler(request, layername).get_download_response() + + +@login_required +def dataset_granule_remove(request, granule_id, layername, template="datasets/dataset_granule_remove.html"): + try: + layer = _resolve_dataset(request, layername, "base.delete_resourcebase", _PERMISSION_MSG_DELETE) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not layer: + raise Http404(_("Not found")) + + if request.method == "GET": + return render(request, template, context={"granule_id": granule_id, "dataset": layer}) + if request.method == "POST": + try: + cat = gs_catalog + cat._cache.clear() + store = cat.get_store(layer.name) + coverages = cat.mosaic_coverages(store) + cat.mosaic_delete_granule(coverages["coverages"]["coverage"][0]["name"], store, granule_id) + except Exception as e: + traceback.print_exc() + message = f'{_("Unable to delete layer")}: {layer.alternate}.' + if "referenced by layer group" in getattr(e, "message", ""): + message = _( + "This dataset is a member of a layer group, you must remove the dataset from the group " + "before deleting." + ) + + messages.error(request, message) + return render(request, template, context={"layer": layer}) + return HttpResponseRedirect(layer.get_absolute_url()) + else: + return HttpResponse("Not allowed", status=403) + + +def get_dataset(request, layername): + """Get Dataset object as JSON""" + + # Function to treat Decimal in json.dumps. + # http://stackoverflow.com/a/16957370/1198772 + def decimal_default(obj): + if isinstance(obj, decimal.Decimal): + return float(obj) + raise TypeError + + logger.debug("Call get layer") + if request.method == "GET": + try: + dataset_obj = _resolve_dataset(request, layername) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not dataset_obj: + raise Http404(_("Not found")) + + logger.debug(layername) + response = { + "typename": layername, + "name": dataset_obj.name, + "title": dataset_obj.title, + "url": dataset_obj.get_tiles_url(), + "bbox_string": dataset_obj.bbox_string, + "bbox_x0": dataset_obj.bbox_helper.xmin, + "bbox_x1": dataset_obj.bbox_helper.xmax, + "bbox_y0": dataset_obj.bbox_helper.ymin, + "bbox_y1": dataset_obj.bbox_helper.ymax, + } + return HttpResponse( + json.dumps(response, ensure_ascii=False, default=decimal_default), content_type="application/javascript" + ) + + +def dataset_metadata_detail(request, layername, template="datasets/dataset_metadata_detail.html", custom_metadata=None): + try: + layer = _resolve_dataset(request, layername, "view_resourcebase", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not layer: + raise Http404(_("Not found")) + + group = None + if layer.group: + try: + group = GroupProfile.objects.get(slug=layer.group.name) + except GroupProfile.DoesNotExist: + group = None + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + + register_event(request, "view_metadata", layer) + perms_list = layer.get_user_perms(request.user) + + return render( + request, + template, + context={ + "resource": layer, + "perms_list": perms_list, + "group": group, + "SITEURL": site_url, + "custom_metadata": custom_metadata, + }, + ) + + +def dataset_metadata_upload(request, layername, template="datasets/dataset_metadata_upload.html"): + try: + layer = _resolve_dataset(request, layername, "base.change_resourcebase", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not layer: + raise Http404(_("Not found")) + + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + return render(request, template, context={"resource": layer, "layer": layer, "SITEURL": site_url}) + + +def dataset_sld_upload(request, layername, template="datasets/dataset_style_upload.html"): + try: + layer = _resolve_dataset(request, layername, "base.change_resourcebase", _PERMISSION_MSG_METADATA) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not layer: + raise Http404(_("Not found")) + + site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL + return render(request, template, context={"resource": layer, "dataset": layer, "SITEURL": site_url}) + + +@xframe_options_exempt +def dataset_embed(request, layername): + try: + layer = _resolve_dataset(request, layername, "base.view_resourcebase", _PERMISSION_MSG_VIEW) + except PermissionDenied: + return HttpResponse(_("Not allowed"), status=403) + except Exception: + raise Http404(_("Not found")) + if not layer: + raise Http404(_("Not found")) + + # Update count for popularity ranking, + # but do not includes admins or resource owners + layer.view_count_up(request.user) + + access_token = None + if request and request.user: + access_token = get_or_create_token(request.user) + if access_token and not access_token.is_expired(): + access_token = access_token.token + else: + access_token = None + + context_dict = { + "access_token": access_token, + "resource": layer, + } + return TemplateResponse(request, "datasets/dataset_embed.html", context=context_dict) + + +@login_required +def dataset_batch_metadata(request): + return batch_modify(request, "Dataset") + + +def dataset_view_counter(dataset_id, viewer): + _l = Dataset.objects.get(id=dataset_id) + _u = get_user_model().objects.get(username=viewer) + _l.view_count_up(_u, do_local=True) diff --git a/geonode/loaders.py b/geonode/loaders.py new file mode 100644 index 0000000..40de67c --- /dev/null +++ b/geonode/loaders.py @@ -0,0 +1,34 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django import db +from djcelery.loaders import DjangoLoader + + +class GeoNodeCeleryTaksLoader(DjangoLoader): + def on_task_init(self, task_id, task): + """Called before every task.""" + for conn in db.connections.all(): + try: + if not conn.in_atomic_block and ( + not conn.connection or (conn.connection.cursor() and not conn.is_usable()) + ): + conn.close() + except Exception: + pass + super().on_task_init(task_id, task) diff --git a/geonode/local_settings.py.geoserver.sample b/geonode/local_settings.py.geoserver.sample new file mode 100644 index 0000000..d7ea4c9 --- /dev/null +++ b/geonode/local_settings.py.geoserver.sample @@ -0,0 +1,489 @@ +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +""" There are 3 ways to override GeoNode settings: + 1. Using environment variables, if your changes to GeoNode are minimal. + 2. Creating a downstream project, if you are doing a lot of customization. + 3. Override settings in a local_settings.py file, legacy. +""" + +import ast +import os + +try: # python2 + from urlparse import urlparse, urlunparse, urlsplit, urljoin +except ImportError: + # Python 3 fallback + from urllib.parse import urlparse, urlunparse, urlsplit, urljoin +from geonode.settings import * + +PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) + +MEDIA_ROOT = os.getenv("MEDIA_ROOT", os.path.join(PROJECT_ROOT, "uploaded")) + +STATIC_ROOT = os.getenv("STATIC_ROOT", os.path.join(PROJECT_ROOT, "static_root")) + +TIME_ZONE = "UTC" + +# Login and logout urls override +LOGIN_URL = os.getenv("LOGIN_URL", "{}account/login/".format(SITEURL)) +LOGOUT_URL = os.getenv("LOGOUT_URL", "{}account/logout/".format(SITEURL)) + +ACCOUNT_LOGIN_REDIRECT_URL = os.getenv("LOGIN_REDIRECT_URL", SITEURL) +ACCOUNT_LOGOUT_REDIRECT_URL = os.getenv("LOGOUT_REDIRECT_URL", SITEURL) + +AVATAR_GRAVATAR_SSL = ast.literal_eval(os.getenv("AVATAR_GRAVATAR_SSL", "True")) + +# Backend +DATABASES = { + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": "geonode", + "USER": "geonode", + "PASSWORD": "geonode", + "HOST": "localhost", + "PORT": "5432", + "CONN_MAX_AGE": 0, + "CONN_TOUT": 5, + "OPTIONS": { + "connect_timeout": 5, + }, + }, + # vector datastore for uploads + "datastore": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + # 'ENGINE': '', # Empty ENGINE name disables + "NAME": "geonode_data", + "USER": "geonode", + "PASSWORD": "geonode", + "HOST": "localhost", + "PORT": "5432", + "CONN_MAX_AGE": 0, + "CONN_TOUT": 5, + "OPTIONS": { + "connect_timeout": 5, + }, + }, +} + +GEOSERVER_LOCATION = os.getenv("GEOSERVER_LOCATION", "http://localhost:8080/geoserver/") + +GEOSERVER_PUBLIC_HOST = os.getenv("GEOSERVER_PUBLIC_HOST", SITE_HOST_NAME) + +GEOSERVER_PUBLIC_PORT = os.getenv("GEOSERVER_PUBLIC_PORT", 8080) + +_default_public_location = ( + "http://{}:{}/geoserver/".format(GEOSERVER_PUBLIC_HOST, GEOSERVER_PUBLIC_PORT) + if GEOSERVER_PUBLIC_PORT + else "http://{}/geoserver/".format(GEOSERVER_PUBLIC_HOST) +) + +GEOSERVER_WEB_UI_LOCATION = os.getenv("GEOSERVER_WEB_UI_LOCATION", GEOSERVER_LOCATION) + +GEOSERVER_PUBLIC_LOCATION = os.getenv("GEOSERVER_PUBLIC_LOCATION", _default_public_location) + +GEOSERVER_ADMIN_USER = os.getenv("GEOSERVER_ADMIN_USER", "admin") + +GEOSERVER_ADMIN_PASSWORD = os.getenv("GEOSERVER_ADMIN_PASSWORD", "geoserver") + +# OGC (WMS/WFS/WCS) Server Settings +OGC_SERVER = { + "default": { + "BACKEND": "geonode.geoserver", + "LOCATION": GEOSERVER_LOCATION, + "WEB_UI_LOCATION": GEOSERVER_WEB_UI_LOCATION, + "LOGIN_ENDPOINT": "j_spring_oauth2_geonode_login", + "LOGOUT_ENDPOINT": "j_spring_oauth2_geonode_logout", + # PUBLIC_LOCATION needs to be kept like this because in dev mode + # the proxy won't work and the integration tests will fail + # the entire block has to be overridden in the local_settings + "PUBLIC_LOCATION": GEOSERVER_PUBLIC_LOCATION, + "USER": GEOSERVER_ADMIN_USER, + "PASSWORD": GEOSERVER_ADMIN_PASSWORD, + "MAPFISH_PRINT_ENABLED": True, + "PRINT_NG_ENABLED": True, + "GEONODE_SECURITY_ENABLED": True, + "GEOFENCE_SECURITY_ENABLED": True, + "GEOFENCE_TIMEOUT": int(os.getenv("GEOFENCE_TIMEOUT", os.getenv("OGC_REQUEST_TIMEOUT", "60"))), + "WMST_ENABLED": False, + "BACKEND_WRITE_ENABLED": True, + "WPS_ENABLED": False, + "LOG_FILE": "%s/geoserver/data/logs/geoserver.log" % os.path.abspath(os.path.join(PROJECT_ROOT, os.pardir)), + # Set to dictionary identifier of database containing spatial data in DATABASES dictionary to enable + "DATASTORE": "datastore", + "TIMEOUT": int(os.getenv("OGC_REQUEST_TIMEOUT", "60")), + "MAX_RETRIES": int(os.getenv("OGC_REQUEST_MAX_RETRIES", "5")), + "BACKOFF_FACTOR": float(os.getenv("OGC_REQUEST_BACKOFF_FACTOR", "0.3")), + "POOL_MAXSIZE": int(os.getenv("OGC_REQUEST_POOL_MAXSIZE", "10")), + "POOL_CONNECTIONS": int(os.getenv("OGC_REQUEST_POOL_CONNECTIONS", "10")), + } +} + +# If you want to enable Mosaics use the following configuration +UPLOADER = { + "BACKEND": "geonode.importer", + "OPTIONS": { + "TIME_ENABLED": True, + "MOSAIC_ENABLED": False, + }, + "SUPPORTED_CRS": ["EPSG:4326", "EPSG:3785", "EPSG:3857", "EPSG:32647", "EPSG:32736"], + "SUPPORTED_EXT": [".shp", ".csv", ".kml", ".kmz", ".json", ".geojson", ".tif", ".tiff", ".geotiff", ".gml", ".xml"], +} + +# CSW settings +CATALOGUE = { + "default": { + # The underlying CSW implementation + # default is pycsw in local mode (tied directly to GeoNode Django DB) + "ENGINE": "geonode.catalogue.backends.pycsw_local", + # pycsw in non-local mode + # 'ENGINE': 'geonode.catalogue.backends.pycsw_http', + # deegree and others + # 'ENGINE': 'geonode.catalogue.backends.generic', + # The FULLY QUALIFIED base url to the CSW instance for this GeoNode + "URL": urljoin(SITEURL, "/catalogue/csw"), + # 'URL': 'http://localhost:8080/deegree-csw-demo-3.0.4/services', + # 'ALTERNATES_ONLY': True, + } +} + +# pycsw settings +PYCSW = { + # pycsw configuration + "CONFIGURATION": { + # uncomment / adjust to override server config system defaults + # 'server': { + # 'maxrecords': '10', + # 'pretty_print': 'true', + # 'federatedcatalogues': 'http://catalog.data.gov/csw' + # }, + "server": { + "home": ".", + "url": CATALOGUE["default"]["URL"], + "encoding": "UTF-8", + "language": LANGUAGE_CODE, + "maxrecords": "20", + "pretty_print": "true", + # 'domainquerytype': 'range', + "domaincounts": "true", + "profiles": "apiso,ebrim", + }, + "manager": { + # authentication/authorization is handled by Django + "transactions": "false", + "allowed_ips": "*", + # 'csw_harvest_pagesize': '10', + }, + "metadata:main": { + "identification_title": "GeoNode Catalogue", + "identification_abstract": "GeoNode is an open source platform" + " that facilitates the creation, sharing, and collaborative use" + " of geospatial data", + "identification_keywords": "sdi, catalogue, discovery, metadata," " GeoNode", + "identification_keywords_type": "theme", + "identification_fees": "None", + "identification_accessconstraints": "None", + "provider_name": "Organization Name", + "provider_url": SITEURL, + "contact_name": "Lastname, Firstname", + "contact_position": "Position Title", + "contact_address": "Mailing Address", + "contact_city": "City", + "contact_stateorprovince": "Administrative Area", + "contact_postalcode": "Zip or Postal Code", + "contact_country": "Country", + "contact_phone": "+xx-xxx-xxx-xxxx", + "contact_fax": "+xx-xxx-xxx-xxxx", + "contact_email": "Email Address", + "contact_url": "Contact URL", + "contact_hours": "Hours of Service", + "contact_instructions": "During hours of service. Off on " "weekends.", + "contact_role": "pointOfContact", + }, + "metadata:inspire": { + "enabled": "true", + "languages_supported": "eng,gre", + "default_language": "eng", + "date": "YYYY-MM-DD", + "gemet_keywords": "Utility and governmental services", + "conformity_service": "notEvaluated", + "contact_name": "Organization Name", + "contact_email": "Email Address", + "temp_extent": "YYYY-MM-DD/YYYY-MM-DD", + }, + } +} + +# -- START Client Hooksets Setup + +# GeoNode javascript client configuration + +# default map projection +# Note: If set to EPSG:4326, then only EPSG:4326 basemaps will work. +DEFAULT_MAP_CRS = os.environ.get("DEFAULT_MAP_CRS", "EPSG:3857") + +DEFAULT_LAYER_FORMAT = os.environ.get("DEFAULT_LAYER_FORMAT", "image/png") + +# Where should newly created maps be focused? +DEFAULT_MAP_CENTER = (os.environ.get("DEFAULT_MAP_CENTER_X", 0), os.environ.get("DEFAULT_MAP_CENTER_Y", 0)) + +# How tightly zoomed should newly created maps be? +# 0 = entire world; +# maximum zoom is between 12 and 15 (for Google Maps, coverage varies by area) +DEFAULT_MAP_ZOOM = int(os.environ.get("DEFAULT_MAP_ZOOM", 3)) + +MAPBOX_ACCESS_TOKEN = os.environ.get("MAPBOX_ACCESS_TOKEN", None) +BING_API_KEY = os.environ.get("BING_API_KEY", None) +GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", None) + +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY = os.getenv("GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore") + +MAP_BASELAYERS = [{}] + +""" +MapStore2 REACT based Client parameters +""" +if GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY == "mapstore": + GEONODE_CLIENT_HOOKSET = os.getenv("GEONODE_CLIENT_HOOKSET", "geonode_mapstore_client.hooksets.MapStoreHookSet") + + if "geonode_mapstore_client" not in INSTALLED_APPS: + INSTALLED_APPS += ( + "mapstore2_adapter", + "geonode_mapstore_client", + ) + + def get_geonode_catalogue_service(): + if PYCSW: + pycsw_config = PYCSW["CONFIGURATION"] + if pycsw_config: + pycsw_catalogue = { + ("%s" % pycsw_config["metadata:main"]["identification_title"]): { + "url": CATALOGUE["default"]["URL"], + "type": "csw", + "title": pycsw_config["metadata:main"]["identification_title"], + "autoload": True, + } + } + return pycsw_catalogue + return None + + GEONODE_CATALOGUE_SERVICE = get_geonode_catalogue_service() + + DEFAULT_MS2_BACKGROUNDS = [ + { + "type": "tileprovider", + "title": "Stamen Watercolor", + "provider": "Stamen.Watercolor", + "name": "Stamen.Watercolor", + "source": "Stamen", + "group": "background", + "thumbURL": "https://stamen-tiles-c.a.ssl.fastly.net/watercolor/0/0/0.jpg", + "visibility": False, + }, + { + "type": "tileprovider", + "title": "Stamen Terrain", + "provider": "Stamen.Terrain", + "name": "Stamen.Terrain", + "source": "Stamen", + "group": "background", + "thumbURL": "https://stamen-tiles-d.a.ssl.fastly.net/terrain/0/0/0.png", + "visibility": False, + }, + { + "type": "tileprovider", + "title": "Stamen Toner", + "provider": "Stamen.Toner", + "name": "Stamen.Toner", + "source": "Stamen", + "group": "background", + "thumbURL": "https://stamen-tiles-d.a.ssl.fastly.net/toner/0/0/0.png", + "visibility": False, + }, + { + "type": "osm", + "title": "Open Street Map", + "name": "mapnik", + "source": "osm", + "group": "background", + "visibility": True, + }, + { + "type": "tileprovider", + "title": "OpenTopoMap", + "provider": "OpenTopoMap", + "name": "OpenTopoMap", + "source": "OpenTopoMap", + "group": "background", + "visibility": False, + }, + { + "type": "wms", + "title": "Sentinel-2 cloudless - https://s2maps.eu", + "format": "image/jpeg", + "id": "s2cloudless", + "name": "s2cloudless:s2cloudless", + "url": [ + "https://maps1.geosolutionsgroup.com/geoserver/wms", + "https://maps2.geosolutionsgroup.com/geoserver/wms", + "https://maps3.geosolutionsgroup.com/geoserver/wms", + "https://maps4.geosolutionsgroup.com/geoserver/wms", + "https://maps5.geosolutionsgroup.com/geoserver/wms", + "https://maps6.geosolutionsgroup.com/geoserver/wms", + ], + "group": "background", + "thumbURL": f"{SITEURL}static/mapstorestyle/img/s2cloudless-s2cloudless.png", + "visibility": False, + "credits": { + "title": 'Sentinel-2 cloudless 2016 by EOX IT Services GmbH' + }, + }, + { + "source": "ol", + "group": "background", + "id": "none", + "name": "empty", + "title": "Empty Background", + "type": "empty", + "visibility": False, + "args": ["Empty Background", {"visibility": False}], + }, + ] + + if MAPBOX_ACCESS_TOKEN: + MAPBOX_BASEMAPS = { + "type": "tileprovider", + "title": "MapBox streets-v11", + "provider": "MapBoxStyle", + "name": "MapBox streets-v11", + "accessToken": "%s" % MAPBOX_ACCESS_TOKEN, + "source": "streets-v11", + "thumbURL": "https://api.mapbox.com/styles/v1/mapbox/streets-v11/tiles/256/6/33/23?access_token=%s" + % MAPBOX_ACCESS_TOKEN, + "group": "background", + "visibility": True, + } + DEFAULT_MS2_BACKGROUNDS = [ + MAPBOX_BASEMAPS, + ] + DEFAULT_MS2_BACKGROUNDS + + if BING_API_KEY: + BING_BASEMAPS = [ + { + "type": "bing", + "title": "Bing Aerial", + "name": "AerialWithLabels", + "source": "bing", + "group": "background", + "apiKey": "{{apiKey}}", + "visibility": True, + }, + { + "type": "bing", + "title": "Bing RoadOnDemand", + "name": "RoadOnDemand", + "source": "bing", + "group": "background", + "apiKey": "{{apiKey}}", + "thumbURL": "%sstatic/mapstorestyle/img/bing_road_on_demand.png" % SITEURL, + "visibility": False, + }, + { + "type": "bing", + "title": "Bing AerialWithLabelsOnDemand", + "name": "AerialWithLabelsOnDemand", + "source": "bing", + "group": "background", + "apiKey": "{{apiKey}}", + "thumbURL": "%sstatic/mapstorestyle/img/bing_aerial_w_labels.png" % SITEURL, + "visibility": False, + }, + { + "type": "bing", + "title": "Bing CanvasDark", + "name": "CanvasDark", + "source": "bing", + "group": "background", + "apiKey": "{{apiKey}}", + "thumbURL": "%sstatic/mapstorestyle/img/bing_canvas_dark.png" % SITEURL, + "visibility": False, + }, + ] + DEFAULT_MS2_BACKGROUNDS = [ + BING_BASEMAPS, + ] + DEFAULT_MS2_BACKGROUNDS + + MAPSTORE_BASELAYERS = DEFAULT_MS2_BACKGROUNDS + # MAPSTORE_BASELAYERS_SOURCES allow to configure tilematrix sets for wmts layers + MAPSTORE_BASELAYERS_SOURCES = os.environ.get("MAPSTORE_BASELAYERS_SOURCES", {}) + +# -- END Client Hooksets Setup + +LOGGING = { + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "verbose": {"format": "%(levelname)s %(asctime)s %(module)s %(process)d " "%(thread)d %(message)s"}, + "simple": { + "format": "%(message)s", + }, + }, + "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, + "handlers": { + "console": {"level": "INFO", "class": "logging.StreamHandler", "formatter": "simple"}, + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler", + }, + }, + "loggers": { + "django": { + "handlers": ["console"], + "level": "ERROR", + }, + "geonode": { + "handlers": ["console"], + "level": "INFO", + }, + "geoserver-restconfig.catalog": { + "handlers": ["console"], + "level": "ERROR", + }, + "owslib": { + "handlers": ["console"], + "level": "ERROR", + }, + "pycsw": { + "handlers": ["console"], + "level": "INFO", + }, + "celery": { + "handlers": ["console"], + "level": "ERROR", + }, + }, +} + +# Additional settings +X_FRAME_OPTIONS = "ALLOW-FROM %s" % SITEURL +CORS_ALLOW_ALL_ORIGINS = True + +GEOIP_PATH = "/usr/local/share/GeoIP" diff --git a/geonode/locale/af/LC_MESSAGES/django.mo b/geonode/locale/af/LC_MESSAGES/django.mo new file mode 100644 index 0000000..6fc6d40 Binary files /dev/null and b/geonode/locale/af/LC_MESSAGES/django.mo differ diff --git a/geonode/locale/af/LC_MESSAGES/django.po b/geonode/locale/af/LC_MESSAGES/django.po new file mode 100644 index 0000000..d4c32a8 --- /dev/null +++ b/geonode/locale/af/LC_MESSAGES/django.po @@ -0,0 +1,6945 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Andres Alfonso Lemon Gomez , 2014 +msgid "" +msgstr "" +"Project-Id-Version: GeoNode\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-07-08 09:02+0300\n" +"PO-Revision-Date: 2016-08-23 13:57+0000\n" +"Last-Translator: Julien Collaer \n" +"Language-Team: Afrikaans (http://www.transifex.com/geonode/geonode/language/" +"af/)\n" +"Language: af\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#, fuzzy +#| msgid "party that owns the resource" +msgid "Request to download a resource" +msgstr "party wat die eienaar van die hulpbron" + +#, fuzzy +#| msgid "party that owns the resource" +msgid "A request for downloading a resource was sent" +msgstr "party wat die eienaar van die hulpbron" + +#, fuzzy +#| msgid "Resources" +msgid "Request resource change" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "Set permissions for this resource" +msgid "Owner has requested permissions to modify a resource" +msgstr "Stel regte vir hierdie hulpbron" + +msgid "series" +msgstr "reeks" + +msgid "computer program or routine" +msgstr "rekenaarprogram of roetine" + +#, fuzzy +#| msgid "date type" +msgid "feature type" +msgstr "datum tipe" + +msgid "copy or imitation of an existing or hypothetical object" +msgstr "kopie of nabootsing van 'n bestaande of denkbeeldige voorwerp" + +msgid "collection hardware" +msgstr "versameling hardeware" + +msgid "collection session" +msgstr "versameling sessie" + +msgid "non-geographic data" +msgstr "nie-geografiese data" + +msgid "property type" +msgstr "eiendom tipe" + +msgid "field session" +msgstr "gebied sessie" + +#, fuzzy +#| msgid "dataset" +msgid "dataset" +msgstr "dataset" + +#, fuzzy +#| msgid "Resources" +msgid "service interfaces" +msgstr "Hulpbronne" + +msgid "attribute class" +msgstr "kenmerk klas" + +msgid "characteristic of a feature" +msgstr "kenmerk van 'n funksie" + +msgid "tile or spatial subset of geographic data" +msgstr "teël of ruimtelike subset van geografiese data" + +msgid "feature" +msgstr "funksie" + +msgid "dimension group" +msgstr "dimensie groep" + +msgid "frequency of maintenance for the data is not known" +msgstr "frekwensie van onderhoud vir die data is nie bekend" + +msgid "data is repeatedly and frequently updated" +msgstr "data is herhaaldelik en gereeld opgedateer" + +msgid "there are no plans to update the data" +msgstr "daar is geen planne om die data te verfris" + +msgid "data is updated each day" +msgstr "data word opgedateer elke dag" + +msgid "data is updated every year" +msgstr "data word elke jaar bygewerk" + +msgid "data is updated as deemed necessary" +msgstr "data word opgedateer soos nodig geag" + +msgid "data is updated each month" +msgstr "data word opgedateer elke maand" + +msgid "data is updated every two weeks" +msgstr "data word opgedateer elke twee weke" + +msgid "data is updated in intervals that are uneven in duration" +msgstr "data is opgedateer in intervalle wat ongelyk in duur" + +msgid "data is updated on a weekly basis" +msgstr "data word opgedateer op 'n weeklikse basis" + +msgid "data is updated twice each year" +msgstr "data word twee keer per jaar opgedateer" + +msgid "data is updated every three months" +msgstr "data word elke drie maande opgedateer" + +msgid "No information provided" +msgstr "Geen inligting verskaf" + +msgid "Category is required." +msgstr "Kategorie word vereis." + +msgid "Keywords from Thesaurus" +msgstr "" + +msgid "List of keywords from Thesaurus" +msgstr "" + +msgid "Abstract" +msgstr "Abstrakte" + +msgid "Purpose" +msgstr "Doel" + +#, fuzzy +#| msgid "Notice settings" +msgid "Other constraints" +msgstr "Notice settings" + +#, fuzzy +#| msgid "supplemental information" +msgid "Supplemental information" +msgstr "aanvullende inligting" + +#, fuzzy +#| msgid "data quality statement" +msgid "Data quality statement" +msgstr "data gehalte verklaring" + +msgid "Owner" +msgstr "Eienaar" + +msgid "Date" +msgstr "Datum" + +msgid "temporal extent start" +msgstr "tydelike mate begin" + +msgid "temporal extent end" +msgstr "tydelike mate einde" + +msgid "Person outside GeoNode (fill form)" +msgstr "" + +msgid "Point of Contact" +msgstr "Punt van kontak" + +msgid "Metadata Author" +msgstr "Metadata outeur" + +#, fuzzy +#| msgid "Keywords" +msgid "Free-text Keywords" +msgstr "Soek" + +#, fuzzy +#| msgid "A space or comma-separated list of keywords" +msgid "" +"A space or comma-separated list of keywords. Use the widget to select from " +"Hierarchical tree." +msgstr "'N spasie of komma-geskeide lys van dokumente" + +msgid "Regions" +msgstr "Streke" + +#, fuzzy +#| msgid "Groups" +msgid "Group" +msgstr "Groepe" + +msgid "Category" +msgstr "Kategorie" + +msgid "License" +msgstr "Lisensie" + +msgid "Language" +msgstr "Taal" + +msgid "User" +msgid_plural "Users" +msgstr[0] "" +msgstr[1] "" + +#, fuzzy +#| msgid "Permissions" +msgid "Permission Type" +msgstr "Permissions" + +msgid "Mode" +msgstr "" + +#, fuzzy +#| msgid "Resources" +msgid "Resource" +msgstr "Hulpbronne" + +msgid "Reason" +msgstr "" + +msgid "Short reasoning behind the request" +msgstr "" + +msgid "function performed by the responsible party" +msgstr "funksie wat uitgevoer word deur die verantwoordelike party" + +msgid "brief narrative summary of the content of the resource(s)" +msgstr "kort vertelling opsomming van die inhoud van die hulpbron (s)" + +msgid "reference date for the cited resource" +msgstr "verwysing datum vir die aangehaal hulpbron" + +msgid "identification of when a given event occurred" +msgstr "identifisering van wanneer 'n gegewe gebeurtenis plaasgevind het" + +msgid "version of the cited resource" +msgstr "weergawe van die aangehaal hulpbron" + +msgid "" +"authority or function assigned, as to a ruler, legislative assembly, " +"delegate, or the like." +msgstr "" + +msgid "a DOI will be added by Admin before publication." +msgstr "" + +msgid "summary of the intentions with which the resource(s) was developed" +msgstr "opsomming van die bedoelings waarmee die hulpbron (s) ontwikkel" + +msgid "" +"frequency with which modifications and deletions are made to the data after " +"it is first produced" +msgstr "" +"frekwensie waarmee veranderinge en weglatings is gemaak om die data nadat " +"dit vir die eerste keer vervaardig" + +#, fuzzy +#| msgid "" +#| "commonly used word(s) or formalised word(s) or phrase(s) used to describe " +#| "the subject (space or comma-separated" +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated)" +msgstr "" +"algemeen gebruik woord (e) of geformaliseer woord (e) of frase (s) gebruik " +"word om die vak (ruimte of komma geskei te beskryf" + +#, fuzzy +#| msgid "" +#| "commonly used word(s) or formalised word(s) or phrase(s) used to describe " +#| "the subject (space or comma-separated" +msgid "" +"formalised word(s) or phrase(s) from a fixed thesaurus used to describe the " +"subject (space or comma-separated)" +msgstr "" +"algemeen gebruik woord (e) of geformaliseer woord (e) of frase (s) gebruik " +"word om die vak (ruimte of komma geskei te beskryf" + +msgid "keyword identifies a location" +msgstr "navraag identifiseer 'n plek" + +msgid "limitation(s) placed upon the access or use of the data." +msgstr "beperking (s) geplaas op die toegang tot of gebruik van die data." + +msgid "" +"other restrictions and legal prerequisites for accessing and using the " +"resource or metadata" +msgstr "" +"ander beperkings en wetlike vereistes vir toegang tot en die gebruik van die " +"hulpbronne of metadata" + +msgid "license of the dataset" +msgstr "lisensie van die dataset" + +msgid "language used within the dataset" +msgstr "taal wat gebruik word in die dataset" + +msgid "" +"high-level geographic data thematic classification to assist in the grouping " +"and search of available geographic data sets." +msgstr "" +"hoë-vlak geografiese data tematiese klassifikasie te help in die groepering " +"en soek beskikbaar geografiese data stelle." + +msgid "method used to represent geographic information in the dataset." +msgstr "" +"metode wat gebruik word geografiese inligting in die dataset te " +"verteenwoordig." + +msgid "time period covered by the content of the dataset (start)" +msgstr "tydperk gedek deur die inhoud van die dataset (begin)" + +msgid "time period covered by the content of the dataset (end)" +msgstr "tydperk gedek deur die inhoud van die dataset (einde)" + +msgid "" +"general explanation of the data producer's knowledge about the lineage of a " +"dataset" +msgstr "" +"algemene verduideliking van die data produsent se kennis oor die geslag van " +"'n dataset" + +msgid "title" +msgstr "titel" + +msgid "name by which the cited resource is known" +msgstr "naam wat die aangehaal hulpbron is bekende" + +msgid "abstract" +msgstr "abstrakte" + +msgid "purpose" +msgstr "doel" + +msgid "date" +msgstr "datum" + +msgid "date type" +msgstr "datum tipe" + +msgid "edition" +msgstr "uitgawe" + +#, fuzzy +#| msgid "Attribute" +msgid "Attribution" +msgstr "Kenmerk" + +msgid "DOI" +msgstr "" + +msgid "maintenance frequency" +msgstr "onderhoud frekwensie" + +msgid "keywords" +msgstr "sleutelwoorde" + +msgid "keywords region" +msgstr "sleutelwoorde streek" + +msgid "restrictions" +msgstr "beperkings" + +msgid "restrictions other" +msgstr "Beperkings ander" + +msgid "language" +msgstr "taal" + +#, fuzzy +#| msgid "Spatial Representation Type" +msgid "spatial representation type" +msgstr "Ruimtelike voorstelling Tipe" + +msgid "supplemental information" +msgstr "aanvullende inligting" + +msgid "any other descriptive information about the dataset" +msgstr "enige ander beskrywende inligting oor die dataset" + +msgid "data quality statement" +msgstr "data gehalte verklaring" + +msgid "CSW typename" +msgstr "CSW Type Name" + +msgid "CSW schema" +msgstr "CSW skedule" + +msgid "CSW source" +msgstr "CSW bron" + +msgid "CSW insert date" +msgstr "CSW voeg datum" + +msgid "CSW type" +msgstr "CSW tipe" + +msgid "CSW anytext" +msgstr "CSW anytext" + +msgid "CSW WKT geometry" +msgstr "CSW WKT meetkunde" + +#, fuzzy +#| msgid "Metadata Provider" +msgid "Metadata uploaded preserve" +msgstr "Metadata Verskaffer" + +msgid "Featured" +msgstr "" + +msgid "Should this resource be advertised in home page?" +msgstr "Indien hierdie hulpbron in tuisblad geadverteer word?" + +#, fuzzy +#| msgid "Published From" +msgid "Is Published" +msgstr "Gepubliseer" + +msgid "Should this resource be published and searchable?" +msgstr "Indien hierdie bron gepubliseer word en deursoek?" + +#, fuzzy +#| msgid "Not provided" +msgid "Approved" +msgstr "Nie voorsien" + +msgid "Is this resource validated from a publisher or editor?" +msgstr "" + +#, fuzzy +#| msgid "Thumbnail" +msgid "Thumbnail url" +msgstr "Thumbnail" + +msgid "State" +msgstr "" + +msgid "Hold the resource processing state." +msgstr "" + +msgid "Dirty State" +msgstr "" + +msgid "Security Rules Are Not Synched with GeoServer!" +msgstr "" + +#, fuzzy +#| msgid "Resources" +msgid "Resource Type" +msgstr "Hulpbronne" + +msgid "Metadata" +msgstr "" + +msgid "If true, will be excluded from search" +msgstr "" + +msgid "For example \"kml\"" +msgstr "Byvoorbeeld \"kml\"" + +msgid "For example \"View in Google Earth\"" +msgstr "Byvoorbeeld \"View in Google Earth\"" + +msgid "For example \"text/xml\"" +msgstr "Byvoorbeeld \"text / xml\"" + +msgid "About" +msgstr "Oor" + +#, fuzzy +#| msgid "Owner, Point of Contact, Metadata Author" +msgid "Responsible, Point of Contact, Metadata Author" +msgstr "Eienaar, Point of Contact, Metadata outeur" + +msgid "Responsible" +msgstr "" + +msgid "Title" +msgstr "Titel" + +msgid "SRID" +msgstr "" + +msgid "For more info see" +msgstr "" + +#, fuzzy +#| msgid "Creating a Map" +msgid "Creation Date" +msgstr "Skep van 'n kaart" + +#, fuzzy +#| msgid "Position Name" +msgid "Publication Date" +msgstr "Posisie Naam" + +#, fuzzy +#| msgid "Permission Denied" +msgid "Revision Date" +msgstr "Toestemming geweier" + +msgid "Type" +msgstr "Tipe" + +msgid "Keywords" +msgstr "Soek" + +msgid "More info" +msgstr "" + +msgid "Maintenance Frequency" +msgstr "Onderhoud Frequency" + +msgid "Restrictions" +msgstr "Beperkings" + +msgid "Edition" +msgstr "Uitgawe" + +msgid "Temporal Extent" +msgstr "Tydelike Mate" + +msgid "Data Quality" +msgstr "Data Kwaliteit" + +msgid "Supplemental Information" +msgstr "Laerskool Inligting" + +msgid "Spatial Representation Type" +msgstr "Ruimtelike voorstelling Tipe" + +msgid "No content created yet." +msgstr "" + +#, fuzzy +#| msgid "Temporal Extent" +msgid "Temporal Serie" +msgstr "Tydelike Mate" + +msgid "Select" +msgstr "" + +#, fuzzy +#| msgid "Services" +msgid "Service is" +msgstr "Dienste" + +msgid "online" +msgstr "" + +msgid "offline" +msgstr "" + +msgid "SECURITY NOT YET SYNCHRONIZED" +msgstr "" + +#, fuzzy +#| msgid "Set permissions" +msgid "Sync permissions immediately" +msgstr "Stel regte" + +msgid "PENDING APPROVAL" +msgstr "" + +msgid "UNPUBLISHED" +msgstr "" + +msgid "Create a Map" +msgstr "Skep 'n kaart" + +msgid "View Map" +msgstr "View Map" + +msgid "Batch Edit" +msgstr "" + +msgid "Cancel" +msgstr "Kanselleer" + +msgid "Submit" +msgstr "Stuur" + +#, fuzzy +#| msgid "Set permissions" +msgid "Set Permissions" +msgstr "Stel regte" + +msgid "Discover the available datasets." +msgstr "" + +#, fuzzy +#| msgid "Set Map Thumbnail" +msgid "Upload a Thumbnail" +msgstr "Set Map Thumbnail" + +msgid "Remove (and use auto generated thumbnail)" +msgstr "" + +#, fuzzy +#| msgid "uploaded" +msgid "Upload" +msgstr "foto" + +#, fuzzy +#| msgid "Remote Thumbnail" +msgid "Auto generated thumbnail" +msgstr "Remote Thumbnail" + +#, fuzzy +#| msgid "Create a Map" +msgid "Raster Layer" +msgstr "Skep 'n kaart" + +#, fuzzy +#| msgid "Edit Layer" +msgid "Vector Layer" +msgstr "Wysig Layer" + +#, fuzzy +#| msgid "Temporal Extent" +msgid "Vector Temporal Serie" +msgstr "Tydelike Mate" + +#, fuzzy +#| msgid "Remove this Layer" +msgid "Remote Layer" +msgstr "Remove this Layer" + +#, fuzzy +#| msgid "Create a Map" +msgid "WMS Cascade Layer" +msgstr "Skep 'n kaart" + +msgid "me" +msgstr "" + +msgid "System message: A request to modify resource" +msgstr "" + +#, fuzzy +#| msgid "Set permissions for this resource" +msgid "The resource owner has requested to modify the resource" +msgstr "Stel regte vir hierdie hulpbron" + +#, fuzzy +#| msgid "Resources" +msgid "Resource title" +msgstr "Hulpbronne" + +msgid "Reason for the request" +msgstr "" + +msgid "" +"To allow the change, set the resource to not \"Approved\" under the metadata " +"settingsand write message to the owner to notify him" +msgstr "" + +#, fuzzy +#| msgid "Password changed successfully." +msgid "Resource Cloned Successfully!" +msgstr "Wagwoord suksesvol verander." + +#, python-brace-format +msgid "Error Occurred while Cloning the Resource: {e}" +msgstr "" + +#, fuzzy +#| msgid "Edit Service Metadata" +msgid "Resource Metadata" +msgstr "Wysig Service Metadata" + +msgid "Thumbnail" +msgstr "Thumbnail" + +#, fuzzy +#| msgid "Resources" +msgid "Resource ID" +msgstr "Hulpbronne" + +msgid "Start" +msgstr "" + +msgid "End" +msgstr "" + +msgid "Extent" +msgstr "Mate" + +#, fuzzy +#| msgid "Coordinate Reference System" +msgid "Spatial Reference System Identifier" +msgstr "Koördineer verwysingstelsel" + +msgid "Attributes" +msgstr "Eienskappe" + +#, fuzzy +#| msgid "attribute name" +msgid "Attribute name" +msgstr "kenmerk naam" + +msgid "Description" +msgstr "Beskrywing" + +#, fuzzy +#| msgid "GeoNode Help" +msgid "GeoNode Client Library" +msgstr "GeoNode Hulp" + +msgid "" +"Flora and/or fauna in natural environment. Examples: wildlife, vegetation, " +"biological sciences, ecology, wilderness, sealife, wetlands, habitat." +msgstr "" + +msgid "Biota" +msgstr "" + +msgid "" +"Legal land descriptions. Examples: political and administrative boundaries." +msgstr "" + +msgid "Boundaries" +msgstr "" + +msgid "" +"Processes and phenomena of the atmosphere. Examples: cloud cover, weather, " +"climate, atmospheric conditions, climate change, precipitation." +msgstr "" + +msgid "Climatology Meteorology Atmosphere" +msgstr "" + +msgid "" +"Economic activities, conditions and employment. Examples: production, " +"labour, revenue, commerce, industry, tourism and ecotourism, forestry, " +"fisheries, commercial or subsistence hunting, exploration and exploitation " +"of resources such as minerals, oil and gas." +msgstr "" + +msgid "Economy" +msgstr "" + +msgid "" +"Height above or below sea level. Examples: altitude, bathymetry, digital " +"elevation models, slope, derived products." +msgstr "" + +msgid "Elevation" +msgstr "" + +msgid "" +"Environmental resources, protection and conservation. Examples: " +"environmental pollution, waste storage and treatment, environmental impact " +"assessment, monitoring environmental risk, nature reserves, landscape." +msgstr "" + +#, fuzzy +#| msgid "Announcement" +msgid "Environment" +msgstr "Aankondiging" + +msgid "" +"Rearing of animals and/or cultivation of plants. Examples: agriculture, " +"irrigation, aquaculture, plantations, herding, pests and diseases affecting " +"crops and livestock." +msgstr "" + +msgid "Farming" +msgstr "" + +msgid "" +"Information pertaining to earth sciences. Examples: geophysical features and " +"processes, geology, minerals, sciences dealing with the composition, " +"structure and origin of the earth s rocks, risks of earthquakes, volcanic " +"activity, landslides, gravity information, soils, permafrost, hydrogeology, " +"erosion." +msgstr "" + +#, fuzzy +#| msgid "Supplemental Information" +msgid "Geoscientific Information" +msgstr "Laerskool Inligting" + +msgid "" +"Health, health services, human ecology, and safety. Examples: disease and " +"illness, factors affecting health, hygiene, substance abuse, mental and " +"physical health, health services." +msgstr "" + +msgid "Health" +msgstr "" + +msgid "" +"Base maps. Examples: land cover, topographic maps, imagery, unclassified " +"images, annotations." +msgstr "" + +msgid "Imagery Base Maps Earth Cover" +msgstr "" + +msgid "" +"Inland water features, drainage systems and their characteristics. Examples: " +"rivers and glaciers, salt lakes, water utilization plans, dams, currents, " +"floods, water quality, hydrographic charts." +msgstr "" + +msgid "Inland Waters" +msgstr "" + +msgid "" +"Military bases, structures, activities. Examples: barracks, training " +"grounds, military transportation, information collection." +msgstr "" + +msgid "Intelligence Military" +msgstr "" + +msgid "" +"Positional information and services. Examples: addresses, geodetic networks, " +"control points, postal zones and services, place names." +msgstr "" + +msgid "Location" +msgstr "" + +msgid "" +"Features and characteristics of salt water bodies (excluding inland waters). " +"Examples: tides, tidal waves, coastal information, reefs." +msgstr "" + +msgid "Oceans" +msgstr "" + +msgid "" +"Information used for appropriate actions for future use of the land. " +"Examples: land use maps, zoning maps, cadastral surveys, land ownership." +msgstr "" + +msgid "Planning Cadastre" +msgstr "" + +msgid "" +"Settlements, anthropology, archaeology, education, traditional beliefs, " +"manners and customs, demographic data, recreational areas and activities, " +"social impact assessments, crime and justice, census information. Economic " +"activities, conditions and employment." +msgstr "" + +#, fuzzy +#| msgid "Position Name" +msgid "Population" +msgstr "Posisie Naam" + +msgid "" +"Characteristics of society and cultures. Examples: settlements, " +"anthropology, archaeology, education, traditional beliefs, manners and " +"customs, demographic data, recreational areas and activities, social impact " +"assessments, crime and justice, census information." +msgstr "" + +msgid "Society" +msgstr "" + +msgid "" +"Man-made construction. Examples: buildings, museums, churches, factories, " +"housing, monuments, shops, towers." +msgstr "" + +msgid "Structure" +msgstr "" + +msgid "" +"Means and aids for conveying persons and/or goods. Examples: roads, airports/" +"airstrips, shipping routes, tunnels, nautical charts, vehicle or vessel " +"location, aeronautical charts, railways." +msgstr "" + +#, fuzzy +#| msgid "format" +msgid "Transportation" +msgstr "formaat" + +msgid "" +"Energy, water and waste systems and communications infrastructure and " +"services. Examples: hydroelectricity, geothermal, solar and nuclear sources " +"of energy, water purification and distribution, sewage collection and " +"disposal, electricity and gas distribution, data communication, " +"telecommunication, radio, communication networks." +msgstr "" + +msgid "Utilities Communication" +msgstr "" + +#, fuzzy +#| msgid "Documents related to this map" +msgid "Document Created" +msgstr "Dokumente wat verband hou met die kaart" + +#, fuzzy +#| msgid "This document is related to a " +msgid "A Document was created" +msgstr "Hierdie dokument is wat verband hou met 'n" + +#, fuzzy +#| msgid "Documents" +msgid "Document Updated" +msgstr "Dokumente" + +#, fuzzy +#| msgid "Documents" +msgid "A Document was updated" +msgstr "Dokumente" + +#, fuzzy +#| msgid "Documents" +msgid "Document Approved" +msgstr "Dokumente" + +#, fuzzy +#| msgid "This document is related to a " +msgid "A Document was approved by a Manager" +msgstr "Hierdie dokument is wat verband hou met 'n" + +#, fuzzy +#| msgid "Documents" +msgid "Document Published" +msgstr "Dokumente" + +#, fuzzy +#| msgid "Documents" +msgid "A Document was published" +msgstr "Dokumente" + +#, fuzzy +#| msgid "Documents" +msgid "Document Deleted" +msgstr "Dokumente" + +#, fuzzy +#| msgid "This document is related to a " +msgid "A Document was deleted" +msgstr "Hierdie dokument is wat verband hou met 'n" + +#, fuzzy +#| msgid "Comment on Layer" +msgid "Comment on Document" +msgstr "Kommentaar te lewer op laag" + +#, fuzzy +#| msgid "A map was commented on" +msgid "A Document was commented on" +msgstr "'N kaart is kommentaar op" + +#, fuzzy +#| msgid "Comment on Map" +msgid "Rating for Document" +msgstr "Gradering vir dokument" + +#, fuzzy +#| msgid "A rating was given to a map" +msgid "A rating was given to a document" +msgstr "A-gradering is om 'n kaart gegee" + +msgid "Width" +msgstr "Wydte" + +msgid "Height" +msgstr "Hoogte" + +msgid "Make" +msgstr "" + +msgid "Model" +msgstr "" + +msgid "Latitude" +msgstr "Latitude" + +msgid "Not specified" +msgstr "" + +msgid "Longitude" +msgstr "Longitude" + +msgid "Flash" +msgstr "" + +msgid "True" +msgstr "" + +msgid "False" +msgstr "" + +msgid "Speed Rating" +msgstr "" + +msgid "Link to" +msgstr "Link na" + +msgid "File" +msgstr "Lêer" + +msgid "Document must be a file or url." +msgstr "Dokument moet 'n lêer of url wees." + +msgid "A document cannot have both a file and a url." +msgstr "'N dokument kan nie beide 'n lêer en 'n url." + +msgid "This file type is not allowed" +msgstr "Hierdie lêer tipe is nie toegelaat" + +msgid "Permissions must be valid JSON." +msgstr "Regte moet geldig into wees." + +msgid "The URL of the document if it is external." +msgstr "Die URL van die dokument as dit ekstern." + +msgid "URL" +msgstr "URL" + +msgid "Your browser does not support the audio element." +msgstr "" + +msgid "Your browser does not support the video tag." +msgstr "" + +#, fuzzy +#| msgid "Download" +msgid "Download the" +msgstr "Aflaai" + +#, fuzzy +#| msgid "Documents" +msgid "document" +msgstr "Dokumente" + +msgid "External Resource" +msgstr "Eksterne Hulpbronne" + +msgid "Rate this document" +msgstr "Gradeer hierdie dokument" + +msgid "Average Rating" +msgstr "Gemiddelde gradering" + +msgid "Metadata Detail" +msgstr "" + +msgid "Download Document" +msgstr "Laai dokument" + +msgid "Request Download" +msgstr "" + +msgid "Edit Document" +msgstr "Wysig Document" + +msgid "Request change" +msgstr "" + +msgid "Wizard" +msgstr "" + +msgid "Advanced Edit" +msgstr "" + +msgid "Document" +msgid_plural "Documents" +msgstr[0] "" +msgstr[1] "" + +msgid "Clone" +msgstr "" + +msgid "Replace" +msgstr "" + +msgid "Remove" +msgstr "Verwyder" + +msgid "Close" +msgstr "Close" + +msgid "Download Metadata" +msgstr "Aflaai Metadata" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Full metadata" +msgstr "Aflaai Metadata" + +#, fuzzy +#| msgid "format" +msgid "Text format" +msgstr "formaat" + +#, fuzzy +#| msgid "format" +msgid "HTML format" +msgstr "formaat" + +msgid "Standard Metadata - XML format" +msgstr "" + +#, fuzzy +#| msgid "Resource using this document" +msgid "Resources using this document" +msgstr "Hulpbron gebruik van hierdie dokument" + +msgid "List of resources using this document:" +msgstr "Lys van hulpbronne die gebruik van hierdie dokument:" + +msgid "This document is not related to any maps or layers" +msgstr "Hierdie dokument is nie verwant aan enige kaarte of lae" + +msgid "Permissions" +msgstr "Permissions" + +msgid "Click the button below to change the permissions of this document." +msgstr "Klik op die knoppie onder die regte van hierdie dokument te verander." + +msgid "Change Document Permissions" +msgstr "Verander Document Permissions" + +msgid "Explore Documents" +msgstr "Verken dokumente" + +msgid "Upload Documents" +msgstr "Oplaai dokumente" + +#, python-format +msgid "for %(document_title)s" +msgstr "" + +msgid "Completeness" +msgstr "" + +msgid "Metadata Schema mandatory fields completed" +msgstr "" + +msgid "Check Schema mandatory fields" +msgstr "" + +msgid "Error updating metadata. Please check the following fields: " +msgstr "Fout met die opdatering metadata. Gaan asseblief die volgende velde:" + +msgid "Done" +msgstr "" + +msgid "Metadata Provider" +msgstr "Metadata Verskaffer" + +msgid "<< Back" +msgstr "" + +msgid "Update" +msgstr "Werk" + +#, fuzzy +#| msgid "Next" +msgid "Next >>" +msgstr "Volgende" + +#, fuzzy +#| msgid "Select an attribute" +msgid "Select an option" +msgstr "Kies 'n kenmerk" + +msgid "Edit Metadata" +msgstr "Wysig Metadata" + +msgid "Editing details for" +msgstr "" + +msgid "Return to Document" +msgstr "" + +msgid "Deleting" +msgstr "Skrap" + +msgid "Remove Document" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(document_title)s?\n" +" " +msgstr "" + +msgid "Yes, I am sure" +msgstr "Ja, ek is seker" + +msgid "No, don't remove it" +msgstr "Nee, moenie dit verwyder nie" + +msgid "Replace Document" +msgstr "Vervang Document" + +msgid "Replace " +msgstr "Vervang" + +msgid "Clear" +msgstr "Duidelik" + +#, fuzzy +#| msgid "Upload Documents" +msgid "Upload Document's Thumbnail" +msgstr "Oplaai dokumente" + +msgid "Drop files here" +msgstr "Drop lêers hier" + +msgid " or select them one by one:" +msgstr "of kies hulle een vir een:" + +#, fuzzy +#| msgid "Choose users..." +msgid "Choose Files" +msgstr "Kies gebruikers ..." + +msgid "Files to be uploaded" +msgstr "Lêers opgelaai word" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Is Upload Metadata XML Form" +msgstr "Aflaai Metadata" + +msgid "Upload files" +msgstr "Lêers oplaai" + +msgid "Upload Document" +msgstr "Oplaai Document" + +#, fuzzy +#| msgid "Upload Documents" +msgid "Allowed document types" +msgstr "Oplaai dokumente" + +msgid "Done!" +msgstr "" + +msgid "Edit" +msgstr "" + +#, fuzzy +#| msgid "Ratings" +msgid "Settings" +msgstr "Graderings" + +msgid "Mandatory" +msgstr "" + +#, fuzzy +#| msgid "Time Options" +msgid "Optional" +msgstr "Tyd Options" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "Basic Metadata" +msgstr "Wysig Metadata" + +msgid "Location and Licenses" +msgstr "" + +#, fuzzy +#| msgid "Edit Service Metadata" +msgid "Required Metadata" +msgstr "Wysig Service Metadata" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Optional Metadata" +msgstr "Aflaai Metadata" + +#, fuzzy +#| msgid "This action affects the following maps:" +msgid "Choose one of the following..." +msgstr "Hierdie aksie raak die volgende kaarte:" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Other, Optional, Metadata" +msgstr "Aflaai Metadata" + +msgid "Responsible Parties" +msgstr "" + +#, fuzzy +#| msgid "Change Layer Permissions" +msgid "Responsible and Permissions" +msgstr "Verandering Layer Permissions" + +#, fuzzy +#| msgid "Public" +msgid "Publishing" +msgstr "Openbare" + +msgid "You are not allowed to view this document." +msgstr "Jy word nie toegelaat om hierdie dokument te besigtig." + +msgid "You are not permitted to delete this document" +msgstr "Jy word nie toegelaat om hierdie dokument te verwyder" + +msgid "You do not have permissions for this document." +msgstr "Jy het nie toestemming om hierdie dokument." + +msgid "You are not permitted to modify this document" +msgstr "Jy word nie toegelaat om hierdie dokument te verander" + +msgid "You are not permitted to modify this document's metadata" +msgstr "Jy word nie toegelaat metadata hierdie dokument te verander" + +msgid "You are not permitted to view this document" +msgstr "Jy word nie toegelaat om hierdie dokument te sien" + +#, fuzzy +#| msgid "Not provided" +msgid "Not allowed" +msgstr "Nie voorsien" + +#, fuzzy +#| msgid "Page Not Found" +msgid "Not found" +msgstr "Bladsy nie gevind" + +#, fuzzy +#| msgid "Set permissions for this resource" +msgid "You must set a point of contact for this resource" +msgstr "Stel regte vir hierdie hulpbron" + +#, fuzzy +#| msgid "Set permissions for this resource" +msgid "You must set an author for this resource" +msgstr "Stel regte vir hierdie hulpbron" + +msgid "Favorite" +msgstr "" + +msgid "Add to Favorites" +msgstr "Add to Favorites" + +#, fuzzy +#| msgid "Add to Favorites" +msgid "Go to Favorites" +msgstr "Add to Favorites" + +msgid "Log in to add/delete Favorites." +msgstr "" + +msgid "Delete from Favorites" +msgstr "" + +msgid "Favorites" +msgstr "" + +msgid "Favorites for" +msgstr "" + +msgid "Item" +msgstr "" + +#, fuzzy +#| msgid "Map Created" +msgid "App Created" +msgstr "Kaart geskep" + +#, fuzzy +#| msgid "A map was commented on" +msgid "A App was created" +msgstr "'N kaart is kommentaar op" + +#, fuzzy +#| msgid "Map Created" +msgid "App Updated" +msgstr "Kaart geskep" + +#, fuzzy +#| msgid "last updated on" +msgid "A App was updated" +msgstr "laas op" + +#, fuzzy +#| msgid "Not provided" +msgid "App Approved" +msgstr "Nie voorsien" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A App was approved by a Manager" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "Published From" +msgid "App Published" +msgstr "Gepubliseer" + +#, fuzzy +#| msgid "Published From" +msgid "A App was published" +msgstr "Gepubliseer" + +#, fuzzy +#| msgid "Map Created" +msgid "App Deleted" +msgstr "Kaart geskep" + +#, fuzzy +#| msgid "A map was commented on" +msgid "A App was deleted" +msgstr "'N kaart is kommentaar op" + +#, fuzzy +#| msgid "Comment on Map" +msgid "Comment on App" +msgstr "Kommentaar op kaart" + +#, fuzzy +#| msgid "A map was commented on" +msgid "An App was commented on" +msgstr "'N kaart is kommentaar op" + +#, fuzzy +#| msgid "Rating for Map" +msgid "Rating for App" +msgstr "Waardering vir kaart" + +#, fuzzy +#| msgid "A rating was given to a map" +msgid "A rating was given to an App" +msgstr "A-gradering is om 'n kaart gegee" + +msgid "Name" +msgstr "Naam" + +msgid "zoom" +msgstr "Klik op" + +#, fuzzy +#| msgid "projection" +msgid "projection" +msgstr "projeksie" + +msgid "center X" +msgstr "sentrum X" + +msgid "center Y" +msgstr "sentrum Y" + +msgid "Site URL" +msgstr "Webwerf URL" + +#, fuzzy, python-format +#| msgid "Resources" +msgid "%(resource.type)s" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "Rate this Map" +msgid "Rate this" +msgstr "Gradeer hierdie Map" + +#, fuzzy +#| msgid "Editing details for " +msgid "Editing Tools" +msgstr "Redigering besonderhede vir" + +msgid "Set" +msgstr "" + +msgid "View" +msgstr "View" + +msgid "Specify which users can view or modify this map" +msgstr "Spesifiseer wat gebruikers kan sien of die kaart verander" + +#, fuzzy +#| msgid "Change Permissions of this Map" +msgid "Change Permissions of this" +msgstr "Verandering regte van die kaart" + +msgid "Embed Iframe" +msgstr "" + +msgid "" +"To embed this map, add the following code snippet and customize its " +"properties (scrolling, width, height) based on your needs to your site" +msgstr "" + +msgid "Download" +msgstr "Aflaai" + +#, python-format +msgid "%(GEONODE_APPS_NAME)s" +msgstr "" + +#, fuzzy +#| msgid "Create a New Map" +msgid "Create New" +msgstr "Skep 'n nuwe gids" + +#, fuzzy +#| msgid "GeoExplorer" +msgid "Explore" +msgstr "GeoExplorer" + +#, python-format +msgid "for %(map_title)s" +msgstr "" + +#, fuzzy +#| msgid "" +#| "Note: this map's orginal metadata was populated by importing a metadata " +#| "XML file.\n" +#| " GeoNode's metadata import supports a subset of ISO, FGDC, and " +#| "Dublin Core metadata elements.\n" +#| " Some of your original metadata may have been lost." +msgid "" +"Note: this geoapp's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" +"Let wel: die kaart se oorspronklike metadata is bevolk deur die invoer van " +"'n metadata XML-lêer. \n" +"GeoNode se metadata invoer ondersteun 'n subset van die ISO, Fgdc en Dublin " +"Core metadata elemente. \n" +"Sommige van jou oorspronklike metadata verlore gewees het." + +#, fuzzy +#| msgid "Explore Maps" +msgid "Explore Apps" +msgstr "Verken Maps" + +#, python-format +msgid "" +"\n" +" Editing details for %(map_title)s\n" +" " +msgstr "" +"\n" +"Redigering besonderhede vir %(map_title)s" + +#, fuzzy +#| msgid "Return to" +msgid "Return to GeoApp" +msgstr "Terug te keer na" + +#, fuzzy +#| msgid "View Map" +msgid "New Map" +msgstr "View Map" + +#, fuzzy, python-format +#| msgid "" +#| "\n" +#| " Are you sure you want to remove the group: %(group_title)s?\n" +#| " " +msgid "" +"\n" +" Are you sure you want to remove %(resource_title)s?\n" +" " +msgstr "" +"\n" +"Is jy seker jy wil die groep te verwyder: %(group_title)s?" + +#, fuzzy +#| msgid "Notice settings" +msgid "Other Settings" +msgstr "Notice settings" + +#, fuzzy +#| msgid "You are not permitted to delete this map." +msgid "You are not permitted to delete this app." +msgstr "Jy word nie toegelaat om die kaart te verwyder." + +#, fuzzy +#| msgid "You do not have permissions for this map." +msgid "You do not have permissions for this app." +msgstr "Jy het nie toestemming om hierdie kaart." + +#, fuzzy +#| msgid "You must be logged in to save this map" +msgid "You must be logged in to save this app" +msgstr "Jy moet aangeteken word om die kaart te red" + +#, fuzzy +#| msgid "You are not permitted to save or edit this map." +msgid "You are not permitted to save or edit this app." +msgstr "Jy word nie toegelaat om op te slaan of die kaart te wysig." + +#, fuzzy +#| msgid "You are not allowed to modify this map's metadata." +msgid "You are not allowed to modify this app's metadata." +msgstr "Jy word nie toegelaat om die kaart se metadata te verander." + +#, fuzzy +#| msgid "You are not allowed to view this map." +msgid "You are not allowed to view this app." +msgstr "Jy word nie toegelaat om die kaart te sien." + +msgid "An unknown error has occured." +msgstr "'N Onbekende fout het voorgekom." + +msgid "Layer Uploaded" +msgstr "Laag Uploaded" + +#, fuzzy +#| msgid "A layer was uploaded" +msgid "A layer was uploaded" +msgstr "'N laag is opgelaai" + +msgid "Comment on Layer" +msgstr "Kommentaar te lewer op laag" + +msgid "A layer was commented on" +msgstr "'N laag is kommentaar op" + +msgid "Rating for Layer" +msgstr "Waardering vir laag" + +msgid "A rating was given to a layer" +msgstr "A-gradering is om 'n laag gegee" + +msgid "Points" +msgstr "" + +msgid "Lines" +msgstr "" + +msgid "Polygons" +msgstr "" + +#, fuzzy +#| msgid "dataset" +msgid "Dataset name" +msgstr "dataset" + +#, fuzzy +#| msgid "Attributes" +msgid "Dataset title" +msgstr "Eienskappe" + +#, fuzzy +#| msgid "property type" +msgid "Geometry type" +msgstr "eiendom tipe" + +#, fuzzy +#| msgid "feature type" +msgid "Create Dataset" +msgstr "funksie tipe" + +#, fuzzy +#| msgid "Explore Layers" +msgid "Explore Datasets" +msgstr "Vind Lae" + +#, fuzzy +#| msgid "Create a map using this layer" +msgid "Create an empty dataset" +msgstr "Skep 'n kaart met behulp van hierdie laag" + +#, fuzzy +#| msgid "Attribute" +msgid "Add Attribute" +msgstr "Kenmerk" + +msgid "Create" +msgstr "Skep" + +msgid "String" +msgstr "" + +#, fuzzy +#| msgid "Intervals" +msgid "Integer" +msgstr "Intervalle" + +msgid "Float" +msgstr "" + +#, fuzzy +#| msgid "Not provided" +msgid "No title provided" +msgstr "Nie voorsien" + +#, fuzzy +#| msgid "Not provided" +msgid "No abstract provided" +msgstr "Nie voorsien" + +msgid "An exception occurred loading data to PostGIS" +msgstr "" + +msgid " Additionally an error occured during database cleanup" +msgstr "" + +msgid "Invalid Projection. Layer is missing CRS!" +msgstr "" + +msgid "GZIP" +msgstr "" + +msgid "GeoTIFF" +msgstr "" + +msgid "Zipped Shapefile" +msgstr "Zipped Shape File" + +msgid "GML 2.0" +msgstr "GML 2.0" + +msgid "GML 3.1.1" +msgstr "GML 3.1.1" + +msgid "CSV" +msgstr "CSV" + +msgid "Excel" +msgstr "Excel" + +msgid "GeoJSON" +msgstr "GeoJSON" + +msgid "JPEG" +msgstr "JPEG" + +msgid "PDF" +msgstr "PDF" + +msgid "PNG" +msgstr "PNG" + +msgid "You don't have permissions to change style for this layer" +msgstr "Jy het nie regte styl te verander vir hierdie laag" + +msgid "Bad HTTP Authorization Credentials." +msgstr "Bad HTTP Toestemming papiere." + +msgid "" +"a short version of the name consisting only of letters, numbers, underscores " +"and hyphens." +msgstr "" +"'n kort weergawe van die naam wat slegs uit letters, getalle, onderstreping " +"en koppeltekens." + +msgid "A group already exists with that slug." +msgstr "'N Groep bestaan ​​reeds met die slak." + +msgid "A group already exists with that name." +msgstr "'N Groep bestaan ​​reeds met daardie naam." + +#, fuzzy +#| msgid "User Activities" +msgid "User Identifiers" +msgstr "Gebruikers Aktiwiteite" + +msgid "Assign manager role" +msgstr "" + +#, python-format +msgid "" +"The following are not valid usernames: %(errors)s; not added to the group" +msgstr "" + +#, fuzzy +#| msgid "Categories" +msgid "Group Categories" +msgstr "Kategorieë" + +#, fuzzy +#| msgid "Public" +msgid "Public" +msgstr "Openbare" + +msgid "Public (invite-only)" +msgstr "Openbare (nooi slegs-)" + +msgid "Private" +msgstr "Private" + +msgid "" +"Public: Any registered user can view and join a public group.
          Public " +"(invite-only):Any registered user can view the group. Only invited users " +"can join.
          Private: Registered users cannot see any details about the " +"group, including membership. Only invited users can join." +msgstr "" +"Publiek: Enige geregistreerde gebruiker kan sien en sluit aan by 'n openbare " +"groep.
          Openbare (slegs nooi-) Enige geregistreerde gebruiker kan die " +"groep sien. Slegs genooide gebruikers kan aansluit.
          Privaat: " +"Geregistreerde gebruikers kan sien nie enige besonderhede oor die groep, " +"insluitend lidmaatskap. Slegs genooide gebruikers kan aansluit." + +msgid "" +"Email used to contact one or all group members, such as a mailing list, " +"shared email, or exchange group." +msgstr "" +"E-pos gebruik om een ​​of al die groeplede, soos 'n e-lys, gedeel e-pos, of " +"ruil groep te kontak." + +msgid "Logo" +msgstr "" + +msgid "Email" +msgstr "" + +msgid "A space or comma-separated list of keywords" +msgstr "'N spasie of komma-geskeide lys van dokumente" + +#, fuzzy +#| msgid "Success" +msgid "Access" +msgstr "Sukses" + +msgid "Categories" +msgstr "Kategorieë" + +msgid "Manager" +msgstr "Bestuurder" + +msgid "Member" +msgstr "Lid" + +msgid "Activity Feed for" +msgstr "Aktiwiteit voer vir" + +msgid "All" +msgstr "Alle" + +msgid "Layers" +msgstr "Lae" + +msgid "Maps" +msgstr "Kaarte" + +msgid "Documents" +msgstr "Dokumente" + +msgid "Comments" +msgstr "Kommentaar" + +msgid "No actions yet" +msgstr "Geen aksies nog" + +#, fuzzy +#| msgid "Explore Groups" +msgid "Explore Group Categories" +msgstr "Verken Groepe" + +#, fuzzy +#| msgid "Create a Group" +msgid "Create a New Group Category" +msgstr "Skep 'n groep" + +msgid "Explore Groups" +msgstr "Verken Groepe" + +msgid "Create a Group" +msgstr "Skep 'n groep" + +msgid "Update Group" +msgstr "Werk Group" + +msgid "Remove Group" +msgstr "Verwyder groep" + +msgid "Create Group" +msgstr "Skep Groep" + +#, fuzzy +#| msgid "last modified" +msgid "Last Modified" +msgstr "Die laaste wysiging" + +msgid "This group has not created a logo." +msgstr "Hierdie groep het nie geskep om 'n logo." + +msgid "Edit Group Details" +msgstr "Wysig Group Besonderhede" + +msgid "Manage Group Members" +msgstr "Bestuur Groep Lede" + +msgid "Delete this Group" +msgstr "Verwyder hierdie Groep" + +msgid "Group Activities" +msgstr "Groepaktiwiteite" + +#, python-format +msgid "" +"\n" +" This group is %(access)s.\n" +" " +msgstr "" +"\n" +"Hierdie groep is %(access)s." + +msgid "Anyone may join this group." +msgstr "Enigeen kan hierdie groep aan te sluit." + +msgid "Join Group" +msgstr "Sluit Group" + +msgid "Anyone may view this group but membership is by invitation only." +msgstr "" +"Enigeen kan hierdie groep sien, maar lidmaatskap is slegs op uitnodiging." + +msgid "Membership is by invitation only." +msgstr "Lidmaatskap is slegs op uitnodiging." + +msgid "Managers" +msgstr "Bestuurders" + +msgid "Members" +msgstr "Lede" + +msgid "Create a New Group" +msgstr "" + +msgid "Edit Members for" +msgstr "" + +#, fuzzy +#| msgid "Members" +msgid "Current Members" +msgstr "Lede" + +msgid "Promote" +msgstr "" + +msgid "Demote" +msgstr "" + +msgid "Role" +msgstr "" + +#, fuzzy +#| msgid "Add members" +msgid "Add new members" +msgstr "Voeg lede" + +#, fuzzy +#| msgid "Manage Group Members" +msgid "Add Group Members" +msgstr "Bestuur Groep Lede" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove the group: %(group_title)s?\n" +" " +msgstr "" +"\n" +"Is jy seker jy wil die groep te verwyder: %(group_title)s?" + +msgid "Remove Maps/Data from Group" +msgstr "Verwyder kaarte / Data uit Groep" + +msgid "Remove selected maps/data" +msgstr "Verwyder gekies kaarte / data" + +#, fuzzy +#| msgid "Edit Group Details" +msgid "Edit Group Category Details" +msgstr "Wysig Group Besonderhede" + +msgid "Groups" +msgstr "Groepe" + +#, fuzzy +#| msgid "Create Group" +msgid "Create Group Category" +msgstr "Skep Groep" + +#, fuzzy +#| msgid "Create a Group" +msgid "Create a Group Category" +msgstr "Skep 'n groep" + +#, fuzzy +#| msgid "Create Group" +msgid "Edit Group Category" +msgstr "Skep Groep" + +msgid "Save" +msgstr "Save" + +msgid "ready" +msgstr "" + +msgid "updating-harvestable-resources" +msgstr "" + +#, fuzzy +#| msgid "Resources" +msgid "harvesting-resources" +msgstr "Hulpbronne" + +msgid "checking-availability" +msgstr "" + +#, fuzzy +#| msgid "Harvested" +msgid "Harvester name" +msgstr "Geoes" + +msgid "Base URL of the remote service that is to be harvested" +msgstr "" + +msgid "" +"Whether to periodically schedule this harvester to look for resources on the " +"remote service" +msgstr "" + +msgid "" +"How often (in minutes) should new harvesting sessions be automatically " +"scheduled? Setting this value to zero has the same effect as setting " +"`scheduling_enabled` to False " +msgstr "" + +msgid "Whether the remote service is known to be available or not" +msgstr "" + +msgid "" +"How often (in minutes) should the remote service be checked for availability?" +msgstr "" + +msgid "Last time the remote server was checked for availability" +msgstr "" + +msgid "Last time the remote server was checked for harvestable resources" +msgstr "" + +#, fuzzy +#| msgid "version of the cited resource" +msgid "Default owner of harvested resources" +msgstr "weergawe van die aangehaal hulpbron" + +#, fuzzy +#| msgid "Set permissions on selected resources" +msgid "Default access permissions of harvested resources" +msgstr "Stel regte op geselekteerde bronne" + +msgid "" +"Should new resources be harvested automatically without explicit selection?" +msgstr "" + +msgid "" +"Orphan resources are those that have previously been created by means of a " +"harvesting operation but that GeoNode can no longer find on the remote " +"service being harvested. Should these resources be deleted from GeoNode " +"automatically? This also applies to when a harvester configuration is " +"deleted, in which case all of the resources that originated from that " +"harvester are now considered to be orphan." +msgstr "" + +msgid "Date of last update to the harvester configuration." +msgstr "" + +msgid "" +"Harvester class used to perform harvesting sessions. New harvester types can " +"be added by an admin by changing the main GeoNode `settings.py` file" +msgstr "" + +msgid "" +"Configuration specific to each harvester type. Please consult GeoNode " +"documentation on harvesting for more info. This field is mandatory, so at " +"the very least an empty object (i.e. {}) must be supplied." +msgstr "" + +msgid "Periodic task used to configure harvest scheduling" +msgstr "" + +msgid "Periodic task used to check availability of the remote" +msgstr "" + +#, python-brace-format +msgid "Check availability of {self.name}" +msgstr "" + +msgid "updating-harvestable-resource" +msgstr "" + +msgid "harvesting-resource" +msgstr "" + +msgid "" +"Identifier that allows referencing the resource on its remote service in a " +"unique fashion. This is usually automatically filled by the harvester " +"worker. The harvester worker needs to know how to either read or generate " +"this from each remote resource in order to be able to compare the " +"availability of resources between consecutive harvesting sessions." +msgstr "" + +msgid "" +"Type of the resource in the remote service. Each harvester worker knows how " +"to fill this field, in accordance with the resources for which harvesting is " +"supported" +msgstr "" + +#, python-brace-format +msgid "The e-mail address '{email}' has already been invited." +msgstr "" + +#, python-brace-format +msgid "The e-mail address '{email}' has already accepted an invite." +msgstr "" + +#, python-brace-format +msgid "An active user is already using the e-mail address '{email}'" +msgstr "" + +#, fuzzy +#| msgid "email" +msgid "E-mail" +msgstr "e-pos" + +#, python-format +msgid "Invitations successfully sent to '%(email)s'" +msgstr "" + +#, python-format +msgid "" +"Sorry, it was not possible to invite '%(email)s' due to the following isse: " +"%(error)s (%(type)s)" +msgstr "" + +#, fuzzy +#| msgid "Map Created" +msgid "Layer Created" +msgstr "Kaart geskep" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A Layer was created" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "Layer Uploaded" +msgid "Layer Updated" +msgstr "Laag Uploaded" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A Layer was updated" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "Not provided" +msgid "Layer Approved" +msgstr "Nie voorsien" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A Layer was approved by a Manager" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "Published From" +msgid "Layer Published" +msgstr "Gepubliseer" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A Layer was published" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "Layer Default Style" +msgid "Layer Deleted" +msgstr "Laag Default Style" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A Layer was deleted" +msgstr "'N laag is kommentaar op" + +msgid "YYYYMMDD" +msgstr "" + +msgid "YYYYMMDD'T'hhmmss" +msgstr "" + +msgid "YYYYMMDD'T'hhmmss'Z'" +msgstr "" + +msgid "style name" +msgstr "styl naam" + +msgid "sld text" +msgstr "SLD teks" + +msgid "sld version" +msgstr "SLD weergawe" + +msgid "sld url" +msgstr "SLD url" + +msgid "Workspace" +msgstr "" + +#, fuzzy +#| msgid "date type" +msgid "Store" +msgstr "datum tipe" + +#, fuzzy +#| msgid "date type" +msgid "Storetype" +msgstr "datum tipe" + +#, fuzzy +#| msgid "CSW typename" +msgid "Typename" +msgstr "CSW Type Name" + +msgid "Is mosaic?" +msgstr "" + +msgid "Has time?" +msgstr "" + +msgid "Has elevation?" +msgstr "" + +msgid "Time regex" +msgstr "" + +msgid "Elevation regex" +msgstr "" + +msgid "use featureinfo custom template?" +msgstr "" + +msgid "specifies wether or not use a custom GetFeatureInfo template." +msgstr "" + +msgid "featureinfo custom template" +msgstr "" + +msgid "the custom GetFeatureInfo template HTML contents." +msgstr "" + +msgid "File cannot be opened, maybe check the encoding" +msgstr "" + +#, fuzzy +#| msgid "attribute name" +msgid "attribute name" +msgstr "kenmerk naam" + +msgid "name of attribute as stored in shapefile/spatial database" +msgstr "Naam van kenmerk as gestoor in shape file / ruimtelike databasis" + +#, fuzzy +#| msgid "distribution description" +msgid "attribute description" +msgstr "distribution description" + +msgid "description of attribute to be used in metadata" +msgstr "beskrywing van eienskap in metadata gebruik word" + +msgid "attribute label" +msgstr "kenmerk etiket" + +msgid "title of attribute as displayed in GeoNode" +msgstr "titel van eienskap soos vertoon in GeoNode" + +msgid "attribute type" +msgstr "kenmerk tipe" + +msgid "the data type of the attribute (integer, string, geometry, etc)" +msgstr "die data tipe van die eienskap (integer, string, meetkunde, ens)" + +msgid "visible?" +msgstr "sigbaar?" + +msgid "specifies if the attribute should be displayed in identify results" +msgstr "" +"is bepaal as die kenmerk vertoon moet word in die resultate te identifiseer" + +msgid "display order" +msgstr "vertoon om" + +msgid "" +"specifies the order in which attribute should be displayed in identify " +"results" +msgstr "" +"bepaal die volgorde in wat kenmerk moet in resultate identifiseer vertoon " +"word" + +msgid "Label" +msgstr "Etiket" + +msgid "Image" +msgstr "" + +msgid "Video (mp4)" +msgstr "" + +msgid "Video (ogg)" +msgstr "" + +msgid "Video (webm)" +msgstr "" + +msgid "Video (3gp)" +msgstr "" + +msgid "Video (flv)" +msgstr "" + +msgid "Video (YouTube/VIMEO - embedded)" +msgstr "" + +msgid "Audio" +msgstr "" + +msgid "IFRAME" +msgstr "" + +#, fuzzy +#| msgid "date type" +msgid "featureinfo type" +msgstr "datum tipe" + +#, fuzzy +#| msgid "specifies if the attribute should be displayed in identify results" +msgid "" +"specifies if the attribute should be rendered with an HTML widget on " +"GetFeatureInfo template." +msgstr "" +"is bepaal as die kenmerk vertoon moet word in die resultate te identifiseer" + +#, fuzzy +#| msgid "count" +msgid "count" +msgstr "tel" + +msgid "count value for this field" +msgstr "tel waarde vir die veld" + +msgid "min" +msgstr "min" + +msgid "minimum value for this field" +msgstr "minimum waarde vir die veld" + +msgid "max" +msgstr "Max" + +msgid "maximum value for this field" +msgstr "maksimum waarde vir die veld" + +msgid "average" +msgstr "gemiddelde" + +msgid "average value for this field" +msgstr "gemiddelde waarde vir die veld" + +msgid "median" +msgstr "mediaan" + +msgid "median value for this field" +msgstr "mediaan waarde vir die veld" + +msgid "standard deviation" +msgstr "standaardafwyking" + +msgid "standard deviation for this field" +msgstr "standaard afwyking vir die veld" + +msgid "sum" +msgstr "som" + +msgid "sum value for this field" +msgstr "som waarde vir die veld" + +msgid "unique values for this field" +msgstr "unieke waardes vir hierdie veld" + +#, fuzzy +#| msgid "last modified" +msgid "last modified" +msgstr "Die laaste wysiging" + +msgid "date when attribute statistics were last updated" +msgstr "datum wanneer kenmerk statistieke was laas" + +#, fuzzy +#| msgid "Edit Layer" +msgid "Append to Dataset" +msgstr "Wysig Layer" + +#, fuzzy +#| msgid "Return to" +msgid "Return to Dataset" +msgstr "Terug te keer na" + +#, fuzzy +#| msgid "Edit Service Metadata" +msgid "Preserve Metadata XML" +msgstr "Wysig Service Metadata" + +msgid "Select the charset or leave default" +msgstr "Kies die karakterstel of laat verstek" + +msgid "Home" +msgstr "Huis" + +msgid "Change point of contact" +msgstr "Verander punt van kontak" + +msgid "Assign a new point of contact to the layers below:" +msgstr "Ken 'n nuwe punt van kontak met die lae hieronder:" + +msgid "Dataset WMS GetCapabilities document" +msgstr "" + +msgid "Filter Granules" +msgstr "" + +msgid "Active Filter:" +msgstr "" + +msgid "Granule ID" +msgstr "" + +msgid "Bounding Box" +msgstr "" + +msgid "Time" +msgstr "" + +msgid "Actions" +msgstr "" + +#, fuzzy +#| msgid "Preview" +msgid "Granule Preview" +msgstr "Voorskou" + +#, fuzzy +#| msgid "Remove" +msgid "Granule Remove" +msgstr "Verwyder" + +msgid "Dimension" +msgstr "" + +msgid "Enabled" +msgstr "" + +msgid "Regex" +msgstr "" + +msgid "TIME" +msgstr "" + +msgid "ELEVATION" +msgstr "" + +msgid "Attribute Name" +msgstr "Kenmerk Naam" + +msgid "Range" +msgstr "Reeks" + +msgid "Average" +msgstr "Gemiddeld" + +msgid "Median" +msgstr "Mediaan" + +msgid "Standard Deviation" +msgstr "Standaardafwyking" + +#, fuzzy +#| msgid "Rate this layer" +msgid "Rate this dataset" +msgstr "Gradeer hierdie laag" + +msgid "Analyze with" +msgstr "" + +#, fuzzy +#| msgid "Download" +msgid "Download Dataset" +msgstr "Aflaai" + +msgid "Images" +msgstr "" + +msgid "Data" +msgstr "" + +msgid "Click to filter the layer" +msgstr "" + +#, fuzzy +#| msgid "Are you sure you want to log out?" +msgid "Do you want to filter it?" +msgstr "Is jy seker jy wil hê om aan te meld nie?" + +msgid "Loading..." +msgstr "" + +#, fuzzy +#| msgid "Select an attribute" +msgid "Filter by attributes" +msgstr "Kies 'n kenmerk" + +msgid "Match" +msgstr "" + +#, fuzzy +#| msgid "All" +msgid "all" +msgstr "Alle" + +msgid "any" +msgstr "" + +#, fuzzy +#| msgid "The following users:" +msgid "of the following:" +msgstr "Die volgende gebruikers:" + +msgid "Attribute" +msgstr "Kenmerk" + +msgid "Operator" +msgstr "" + +#, fuzzy +#| msgid "feature type" +msgid "Feature limit" +msgstr "funksie tipe" + +msgid "Pick your download format:" +msgstr "" + +msgid "" +"No data available for this resource. Please contact a system administrator " +"or a manager." +msgstr "" + +msgid "Warning" +msgstr "" + +msgid "Edit Layer" +msgstr "Wysig Layer" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Upload Metadata" +msgstr "Aflaai Metadata" + +msgid "Styles" +msgstr "Style" + +msgid "Manage" +msgstr "Bestuur" + +msgid "Layer" +msgid_plural "Layers" +msgstr[0] "" +msgstr[1] "" + +msgid "Append Data" +msgstr "" + +msgid "Edit data" +msgstr "" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "View Dataset" +msgstr "Wysig Metadata" + +#, fuzzy +#| msgid "format" +msgid "Attribute Information" +msgstr "formaat" + +msgid "ISO Feature Catalogue" +msgstr "" + +msgid "Legend" +msgstr "Legende" + +#, fuzzy +#| msgid "Maps using this layer" +msgid "Maps using this dataset" +msgstr "Kaarte met behulp van hierdie laag" + +#, fuzzy +#| msgid "List of maps using this layer:" +msgid "List of maps using this dataset:" +msgstr "Lys van kaarte deur hierdie laag:" + +#, fuzzy +#| msgid "This layer is not currently used in any maps." +msgid "This dataset is not currently used in any maps." +msgstr "Hierdie laag is tans nie gebruik word in enige kaarte." + +#, fuzzy +#| msgid "Create a map using this layer" +msgid "Create a map using this dataset" +msgstr "Skep 'n kaart met behulp van hierdie laag" + +#, fuzzy +#| msgid "Click the button below to generate a new map based on this layer." +msgid "Click the button below to generate a new map based on this dataset." +msgstr "" +"Klik op die knoppie hieronder om 'n nuwe kaart wat gebaseer is op hierdie " +"laag te genereer." + +msgid "Add the dataset to an existing map" +msgstr "" + +#, fuzzy +#| msgid "Click the button below to change the permissions of this layer." +msgid "Click the button below to add the dataset to the selected map." +msgstr "Klik op die knoppie onder die regte van hierdie laag te verander." + +#, fuzzy +#| msgid "Edit Map" +msgid "Add to Map" +msgstr "Wysig Map" + +msgid "Documents related to this layer" +msgstr "Dokumente wat verband hou met hierdie laag" + +msgid "List of documents related to this layer:" +msgstr "Lys van dokumente wat verband hou met hierdie laag:" + +msgid "" +"The following styles are associated with this layer. Choose a style to view " +"it in the preview map." +msgstr "" +"Die volgende style is wat verband hou met hierdie laag. Kies 'n styl om dit " +"te sien in die voorbeeld kaart." + +#, fuzzy +#| msgid "Layer Default Style" +msgid "(default style)" +msgstr "Laag Default Style" + +msgid "No styles associated with this layer" +msgstr "Geen style wat verband hou met hierdie laag" + +msgid "External service layer" +msgstr "Eksterne diens laag" + +msgid "Source" +msgstr "Bron" + +msgid "Refresh Attributes and Statistics of this layer" +msgstr "" + +msgid "" +"Click the button below to allow GeoNode refreshing the list of available " +"Dataset Attributes. If the option 'WPS_ENABLED' has been also set on the " +"backend, it will recalculate their statistics too." +msgstr "" + +msgid "Refresh Attributes and Statistics" +msgstr "" + +#, fuzzy +#| msgid "Create a map using this layer" +msgid "Clear the Server Cache of this layer" +msgstr "Skep 'n kaart met behulp van hierdie laag" + +#, fuzzy +#| msgid "Click the button below to change the permissions of this layer." +msgid "Click the button below to wipe the tile-cache of this layer." +msgstr "Klik op die knoppie onder die regte van hierdie laag te verander." + +msgid "Empty Tiled-Dataset Cache" +msgstr "" + +msgid "Click the button below to change the permissions of this layer." +msgstr "Klik op die knoppie onder die regte van hierdie laag te verander." + +#, fuzzy +#| msgid "Change Layer Permissions" +msgid "Change Dataset Permissions" +msgstr "Verandering Layer Permissions" + +msgid "Remove Mosaic Granules" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove Granule %(granule_id)s of the " +"Mosaic %(layer_title)s?\n" +" " +msgstr "" + +msgid "This action affects the following maps:" +msgstr "Hierdie aksie raak die volgende kaarte:" + +msgid "No maps are using this layer" +msgstr "Geen kaarte gebruik hierdie laag" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Upload Datasets" +msgstr "Aflaai Metadata" + +#, python-format +msgid "for %(layer_title)s" +msgstr "" + +msgid "" +"Note: this layer's orginal metadata was populated and preserved by importing " +"a metadata XML file.\n" +" This metadata cannot be edited." +msgstr "" + +msgid "" +"Note: this layer's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and " +"Dublin Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" + +#, python-format +msgid "" +"\n" +" Editing details for %(layer_title)s\n" +" " +msgstr "" +"\n" +"Redigering besonderhede vir %(layer_title)s" + +msgid "Use a custom template?" +msgstr "" + +msgid "Display Order" +msgstr "Wys Order" + +#, fuzzy +#| msgid "Display Order" +msgid "Display Type" +msgstr "Wys Order" + +#, fuzzy +#| msgid "visible?" +msgid "Visible" +msgstr "sigbaar?" + +msgid "ERROR" +msgstr "" + +msgid "Following items are required:" +msgstr "" + +#, fuzzy +#| msgid "Upload Layer" +msgid "Upload Dataset Metadata" +msgstr "Oplaai Layer" + +msgid "(XML - ISO, FGDC, ebRIM, Dublin Core)" +msgstr "" + +#, fuzzy +#| msgid "feature type" +msgid "Remove Datasets" +msgstr "funksie tipe" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(layer_title)s?\n" +" " +msgstr "" + +#, fuzzy +#| msgid "Replace Layer" +msgid "Replace Dataset" +msgstr "Vervang Layer" + +msgid "Manage Styles" +msgstr "Bestuur Styles" + +#, python-format +msgid "" +"\n" +" Manage Available Styles for " +"%(layer_title)s\n" +" " +msgstr "" +"\n" +"Bestuur beskikbaar style vir " +"%(layer_title)s" + +#, fuzzy +#| msgid "Layer Default Style" +msgid "Dataset Default Style" +msgstr "Laag Default Style" + +msgid "Available styles" +msgstr "Beskikbaar style" + +msgid "" +"Click on an available style in the upper box to assign it to this layer. " +"Selected styles appear in the lower box." +msgstr "" +"Klik op 'n beskikbare styl in die boonste kassie om dit te wys aan hierdie " +"laag. Gekies style verskyn in die laer boks." + +msgid "Update Available Styles" +msgstr "Werk Beskikbaar Styles" + +msgid "Return to Layer" +msgstr "" + +#, fuzzy +#| msgid "Upload Layer Step 2" +msgid "Upload Dataset Style" +msgstr "Oplaai Layer Stap 2" + +msgid "(SLD - Style Layer Descriptor 1.0, 1.1)" +msgstr "" + +msgid "WARNING" +msgstr "" + +msgid "This will most probably overwrite the current default style!" +msgstr "" + +msgid "Preview" +msgstr "Voorskou" + +#, fuzzy +#| msgid "Attributes" +msgid "Dataset Attributes" +msgstr "Eienskappe" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Upload Dataset" +msgstr "Aflaai Metadata" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Upload status" +msgstr "Aflaai Metadata" + +msgid "" +"This file needs additional configuration to complete the upload process. " +"Please click on the button to fill the required configuration" +msgstr "" + +msgid "Delete" +msgstr "Verwyder" + +#, fuzzy +#| msgid "Upload Document" +msgid "Upload process completed" +msgstr "Oplaai Document" + +msgid "" +"The upload process cannot be completed because the original file is no more " +"available" +msgstr "" + +#, fuzzy +#| msgid "Create" +msgid "Created" +msgstr "Skep" + +msgid "Progress" +msgstr "" + +msgid "page" +msgstr "" + +msgid "of" +msgstr "van" + +#, fuzzy +#| msgid "Incomplete Uploads" +msgid "Delete incomplete upload" +msgstr "Onvolledige Uploads" + +#, fuzzy +#| msgid "Are you sure you want to delete this upload?" +msgid "Are you sure you want to remove this incomplete upload?" +msgstr "Is jy seker jy wil hierdie oplaai te verwyder?" + +#, fuzzy +#| msgid "Upload Layer Step 2" +msgid "Upload Dataset Step: Set SRS" +msgstr "Oplaai Layer Stap 2" + +msgid "Provide CRS for " +msgstr "Verskaf CRS vir" + +msgid "Coordinate Reference System" +msgstr "Koördineer verwysingstelsel" + +#, fuzzy +#| msgid "" +#| "\n" +#| " A coordinate reference system for this layer could not be " +#| "determined.\n" +#| " Locate or enter the appropriate ESPG code for this layer below.\n" +#| " One way to do this is do visit:\n" +#| " prj2epsg\n" +#| " and enter the following:\n" +#| " " +msgid "" +"\n" +" A coordinate reference system for this dataset could not be " +"determined.\n" +" Locate or enter the appropriate ESPG code for this dataset " +"below.\n" +" One way to do this is do visit:\n" +" prj2epsg\n" +" and enter the following:\n" +" " +msgstr "" +"\n" +"A koördineer verwysingstelsel vir hierdie laag kon nie vasgestel word nie. " +"Vind of tik die toepaslike ESPG kode vir hierdie laag hieronder. Een manier " +"om dit te doen, is om Besoek gerus: \n" +"prj2epsg \n" +"en tik die volgende:" + +msgid "Native CRS could not be found!" +msgstr "" + +msgid "Provide EPSG code for Source SRS" +msgstr "" + +msgid "EPSG Code (Source SRS)" +msgstr "" + +msgid "Select a Source SRS" +msgstr "" + +msgid "" +"Source SRS EPSG Code is mandatory and represents the native data Spatial " +"Reference System.\n" +"

          \n" +" This must be coherent with the Geometry values (lon/lat " +"coordinates as an instance) stored on the geospatial dataset.\n" +"

          \n" +" If not specified on the geospatial data itself, it must " +"be manually declared by the operator.\n" +"

          \n" +" More information is provided at the bottom of the page " +"in the \"Additional Help\" sections.\n" +" " +msgstr "" + +msgid "Next" +msgstr "Volgende" + +#, fuzzy +#| msgid "Time Options" +msgid "Advanced Options" +msgstr "Tyd Options" + +msgid "Target SRS" +msgstr "" + +msgid "EPSG Code (Target SRS)" +msgstr "" + +#, fuzzy +#| msgid "Select an attribute" +msgid "Select a Target SRS" +msgstr "Kies 'n kenmerk" + +msgid "leave it empty to use only Source SRS" +msgstr "" + +msgid "Additional Help" +msgstr "" + +#, fuzzy +#| msgid "Coordinate Reference System" +msgid "Spatial Reference System" +msgstr "Koördineer verwysingstelsel" + +msgid "" +"A spatial reference system (SRS) or coordinate reference system (CRS) is a " +"coordinate-based local,\n" +" regional or global system used to locate " +"geographical entities. A spatial reference system defines a specific map\n" +" projection, as well as transformations between " +"different spatial reference systems. Spatial reference systems are\n" +" defined by the OGC's Simple feature access using " +"well-known text, and support has been implemented by several\n" +" standards-based geographic information systems. " +"Spatial reference systems can be referred to using a SRID integer,\n" +" including EPSG codes defined by the " +"International Association of Oil and Gas Producers.\n" +" It is specified in ISO 19111:2007 Geographic " +"information—Spatial referencing by coordinates, also published as\n" +" OGC Abstract Specification, Topic 2: Spatial " +"referencing by coordinate." +msgstr "" + +msgid "Identifiers" +msgstr "" + +msgid "" +"\n" +" A Spatial Reference System Identifier (SRID) is a " +"unique value used to unambiguously identify projected, unprojected,\n" +" and local spatial coordinate system definitions. " +"These coordinate systems form the heart of all GIS applications.\n" +"\n" +" Virtually all major spatial vendors have created " +"their own SRID implementation or refer to those of an authority,\n" +" such as the European Petroleum Survey Group (EPSG).\n" +" " +msgstr "" + +msgid "" +"NOTE: As of 2005 the EPSG SRID values are now maintained by the " +"International\n" +" Association of Oil & Gas Producers (OGP) " +"Surveying & Positioning Committee" +msgstr "" + +msgid "" +"\n" +" SRIDs are the primary key for the Open Geospatial " +"Consortium (OGC) spatial_ref_sys metadata table for the Simple\n" +" Features for SQL Specification, Versions 1.1 and " +"1.2, which is defined as follows:\n" +" " +msgstr "" + +msgid "" +"\n" +" In spatially enabled databases (such as IBM DB2, IBM " +"Informix, Microsoft SQL Server, MySQL, Oracle RDBMS, Teradata, PostGIS and\n" +" SQL Anywhere), SRIDs are used to uniquely identify " +"the coordinate systems used to define columns of spatial data or individual\n" +" spatial objects in a spatial column (depending on " +"the spatial implementation). SRIDs are typically associated with a well " +"known\n" +" text (WKT) string definition of the coordinate " +"system (SRTEXT, above). From the Well Known Text Wikipedia page\n" +" " +msgstr "" + +msgid "" +"“A WKT string for a spatial reference system describes the datum, geoid, " +"coordinate system,\n" +" and map projection of the spatial objects”." +msgstr "" + +msgid "" +"\n" +" Here are two common coordinate systems with their " +"EPSG SRID value followed by their well known text:\n" +" " +msgstr "" + +msgid "" +"\n" +" UTM, Zone 17N, NAD27 — SRID 2029\n" +" " +msgstr "" + +msgid "" +"\n" +" WGS84 — SRID 4326\n" +" " +msgstr "" + +msgid "" +"\n" +" SRID values associated with spatial data can be used " +"to constrain spatial operations — for instance, spatial operations cannot be " +"performed\n" +" between spatial objects with differing SRIDs in some " +"systems, or trigger coordinate system transformations between spatial " +"objects in others.\n" +" " +msgstr "" + +msgid "" +"Source SRS EPSG Code is mandatory and represents the native data Spatial " +"Reference System. This must be coherent with the\n" +" Geometry values (lon/lat coordinates as an " +"instance) stored on the geospatial dataset. If not specified on the " +"geospatial data itself, it\n" +" must be manually declared by the operator." +msgstr "" + +msgid "" +"Target SRS EPSG Code is optional. This must be used only if we need to re-" +"project the coordinates from Source SRS to another one.\n" +" " +msgstr "" + +#, fuzzy +#| msgid "Upload Layer Step 2" +msgid "Upload Dataset Step: CSV Field Mapping" +msgstr "Oplaai Layer Stap 2" + +msgid "Geospatial Data" +msgstr "Geospatial Data" + +msgid "" +"Please indicate which attributes contain the latitude and longitude " +"coordinates in the CSV data." +msgstr "" +"Dui asseblief aan watter eienskappe die breedtegraad bevat en breedtegraad " +"in die CSV data." + +#, fuzzy +#| msgid "" +#| "With this data, GeoNode was able to guess which attributes contain the\n" +#| "latitude and longitude coordinates, but please confirm that the correct\n" +#| "attributes are selected below." +msgid "" +"With this data, GeoNode was able to guess which attributes contain the\n" +" latitude and longitude coordinates, but please confirm that " +"the correct\n" +" attributes are selected below." +msgstr "" +"Met hierdie inligting, GeoNode in staat was om te raai wat eienskappe bevat\n" +" die lengte-en breedtegraad, maar bevestig asseblief dat die regte \n" +"eienskappe word hieronder gekies." + +msgid "Select an attribute" +msgstr "Kies 'n kenmerk" + +#, fuzzy +#| msgid "" +#| "We did not detect columns that could be used for the latitude and " +#| "longitude.\n" +#| "Please verify that you have two columns in your csv file that can be used " +#| "for\n" +#| "the latitude and longitude." +msgid "" +"We did not detect columns that could be used for the latitude and " +"longitude.\n" +" Please verify that you have two columns in your csv file that can be " +"used for\n" +" the latitude and longitude." +msgstr "" +"Ons het nie die kolomme wat gebruik kan word vir die lengte-en breedtegraad " +"spoor. \n" +"Maak seker dat jy twee kolomme in die CSV wat gebruik kan word vir \n" +"die lengte-en breedtegraad." + +msgid "Your upload is either complete or you haven't resumed an earlier one." +msgstr "Jou oplaai is óf volledige of jy nog nie 'n vroeëre hervat." + +msgid "Return to" +msgstr "Terug te keer na" + +msgid "Upload Form" +msgstr "Oplaai Form" + +#, fuzzy +#| msgid "Upload Layer Step 2" +msgid "Upload Dataset Step: Time" +msgstr "Oplaai Layer Stap 2" + +msgid "Inspect data for " +msgstr "" + +msgid "Configure as Time-Series" +msgstr "" + +msgid "" +"Toggling this selector allows you to configure (or not) this data as a time " +"series; in this case you will also have to select an attribute\n" +" to drive the time dimension.\n" +"

          \n" +" If GeoNode is not able to parse any of the values for " +"the selected attribute red markers will appear to highlight the problems.\n" +"

          \n" +" More information is provided at the bottom of the page " +"in the \"Additional Help\" sections.\n" +" " +msgstr "" + +msgid "No" +msgstr "Geen" + +msgid "Use an existing timestamp attribute in the data" +msgstr "" + +#, fuzzy +#| msgid "Existing Time Attribute" +msgid "Yes: with an existing Time-Attribute" +msgstr "Bestaande Tyd Kenmerk" + +msgid "" +"Yes: by converting data to a timestamp using standard date/time " +"representation" +msgstr "" + +#, fuzzy +#| msgid "Convert Number (As Year)" +msgid "Convert a number field into a year" +msgstr "Skakel nommer (As Jaar)" + +#, fuzzy +#| msgid "Convert Number (As Year)" +msgid "Yes: by converting a number as Year" +msgstr "Skakel nommer (As Jaar)" + +msgid "" +"Convert data to a timestamp using standard date/time representation or a " +"custom format" +msgstr "" + +msgid "Start Importer" +msgstr "" + +msgid "Date Format" +msgstr "Datum formaat" + +#, fuzzy +#| msgid "Attribute Name" +msgid "Text Attribute Date Format" +msgstr "Kenmerk Naam" + +msgid "Best Guess" +msgstr "" + +msgid "Custom" +msgstr "" + +#, fuzzy +#| msgid "Choose optional end time attribute" +msgid "Optional End-Time attribute" +msgstr "Kies opsionele einde van die tyd kenmerk" + +msgid "Existing Time Attribute" +msgstr "Bestaande Tyd Kenmerk" + +msgid "Convert Text Attribute" +msgstr "Skakel teks Kenmerk" + +msgid "Convert Number (As Year)" +msgstr "Skakel nommer (As Jaar)" + +#, fuzzy +#| msgid "attribute description" +msgid "Time attribute Presentation" +msgstr "skryf beskrywing" + +msgid "List" +msgstr "Lys" + +msgid "of all the distinct time values" +msgstr "van al die verskillende tyd waardes" + +msgid "Intervals" +msgstr "Intervalle" + +msgid "defined by the resolution" +msgstr "gedefinieer deur die resolusie" + +msgid "Continuous Intervals" +msgstr "Deurlopende Intervalle" + +msgid "" +"for data that is frequently updated, resolution describes the frequency of " +"updates" +msgstr "" +"vir data wat gereeld opgedateer, resolusie beskryf die frekwensie van updates" + +msgid "Resolution of time attribute" +msgstr "Resolusie van die tyd kenmerk" + +msgid "Enabling Time" +msgstr "In staat te stel Tyd" + +#, fuzzy +#| msgid "" +#| "A feature can currently support one or two time attributes. If a single\n" +#| " attribute is used, the feature is considered relevant at that single " +#| "point in time. If two\n" +#| " attributes are used, the second attribute represents the end of a " +#| "valid period for the\n" +#| " feature." +msgid "" +"A dataset can support one or two time attributes. If a single\n" +" attribute is used, the dataset is considered to " +"contain data that is valid at single points in time. If two\n" +" attributes are used, the second attribute represents " +"the end of a valid period hence the dataset is considered\n" +" to contain data that is valid at certain periods in " +"time." +msgstr "" +"'N funksie kan ondersteun tans een of twee keer eienskappe. As 'n enkele\n" +" eienskap gebruik word, word die funksie beskou betrokke by daardie enkele " +"punt in die tyd. Indien twee \n" +"eienskappe gebruik word, die tweede eienskap verteenwoordig die einde van 'n " +"geldige tydperk vir \n" +"die funksie." + +msgid "Selecting an Attribute" +msgstr "Kies 'n Kenmerk" + +#, fuzzy +#| msgid "A time attribute can be one of" +msgid "A time attribute can be" +msgstr "'N tyd eienskap kan een van" + +msgid "An existing date" +msgstr "'N bestaande datum" + +msgid "Text that can be converted to a timestamp" +msgstr "Teks wat omgeskakel kan word na 'n tyd stempel" + +msgid "A number representing a year" +msgstr "'N getal wat 'n jaar" + +msgid "" +"\n" +" For text attributes, one can specify a custom format " +"(as part of the \"Advanced Options\") or use the 'best guess' approach which " +"will try to\n" +" automatically translate well-known recognized " +"patterns into valid times.\n" +" " +msgstr "" + +msgid "The 'best guess' will handle date and optional time variants of" +msgstr "" +"Die 'beste raaiskoot \"sal die datum en opsionele tyd variante van hanteer" + +msgid "In terms of the formatting flags noted above, these are" +msgstr "In terme van die uitleg vlae wat hierbo genoem, dit is" + +msgid "Modal Header" +msgstr "" + +msgid "Some text in the modal" +msgstr "" + +msgid " does NOT match any valid ISO-8601 Date-Time string!" +msgstr "" + +msgid "Valid ISO-8601 for" +msgstr "" + +msgid "matches a valid ISO-8601 Date-Time string!" +msgstr "" + +msgid "Not Valid ISO-8601 for" +msgstr "" + +msgid "does NOT match any valid ISO-8601 Date-Time string!" +msgstr "" + +#, fuzzy +#| msgid "Your selections" +msgid "Wrong Selection" +msgstr "Your selections" + +msgid "Please, select one Time Attribute to test!" +msgstr "" + +msgid "" +"Returning to the upload starting page in 5seconds " +msgstr "" +"Terug te keer na die oplaai tuisblad in 5 sekondes" + +msgid " Or just go " +msgstr "Of net gaan" + +msgid "now" +msgstr "nou" + +#, python-brace-format +msgid "You are attempting to {action_type} a vector dataset with a raster." +msgstr "" + +#, python-brace-format +msgid "You are attempting to {action_type} a raster dataset with a vector." +msgstr "" + +#, python-brace-format +msgid "" +"You are attempting to {action_type} a vector dataset with an unknown format." +msgstr "" + +#, python-brace-format +msgid "" +"Please ensure the name is consistent with the file you are trying to " +"{action_type}." +msgstr "" + +msgid "Local GeoNode dataset has no geometry type." +msgstr "" + +#, python-brace-format +msgid "" +"Please ensure there is at least one geometry " +"type that is consistent with the file you are " +"trying to {action_type}." +msgstr "" + +msgid "The selected Dataset does not exists in the catalog." +msgstr "" + +msgid "Please ensure that the dataset structure is consistent " +msgstr "" + +msgid "" +"Some error occurred while trying to access the uploaded schema: {str(e)}" +msgstr "" + +msgid "" +"There was an error while attempting to upload your data. Please try again, " +"or contact and administrator if the problem continues." +msgstr "" +"Daar was 'n fout tydens 'n poging om jou data te laai. Probeer asseblief " +"weer, of kontak en administrateur indien die probleem voortduur." + +msgid "" +"Note: this dataset's orginal metadata was populated and preserved by " +"importing a metadata XML file. This metadata cannot be edited." +msgstr "" + +#, fuzzy +#| msgid "You are not permitted to delete this layer" +msgid "You are not permitted to delete this dataset" +msgstr "Jy word nie toegelaat om hierdie laag te verwyder" + +#, fuzzy +#| msgid "You do not have permissions for this layer." +msgid "You do not have permissions for this dataset." +msgstr "Jy het nie toestemming om hierdie laag." + +#, fuzzy +#| msgid "You are not permitted to modify this layer" +msgid "You are not permitted to modify this dataset" +msgstr "Jy word nie toegelaat om hierdie laag te verander" + +#, fuzzy +#| msgid "You are not permitted to modify this layer's metadata" +msgid "You are not permitted to modify this dataset's metadata" +msgstr "Jy word nie toegelaat om hierdie laag se metadata te verander" + +#, fuzzy +#| msgid "You are not permitted to view this layer" +msgid "You are not permitted to view this dataset" +msgstr "Jy word nie toegelaat om hierdie laag te sien" + +msgid "" +"This dataset is a member of a layer group, you must remove the dataset from " +"the group before deleting." +msgstr "" + +#, fuzzy, python-format +#| msgid "Remote Thumbnail" +msgid "couldn't generate thumbnail: %s" +msgstr "Remote Thumbnail" + +msgid "Map Created" +msgstr "Kaart geskep" + +#, fuzzy +#| msgid "A map was commented on" +msgid "A Map was created" +msgstr "'N kaart is kommentaar op" + +#, fuzzy +#| msgid "Map Created" +msgid "Map Updated" +msgstr "Kaart geskep" + +#, fuzzy +#| msgid "last updated on" +msgid "A Map was updated" +msgstr "laas op" + +#, fuzzy +#| msgid "Not provided" +msgid "Map Approved" +msgstr "Nie voorsien" + +msgid "A Map was approved by a Manager" +msgstr "" + +#, fuzzy +#| msgid "Published From" +msgid "Map Published" +msgstr "Gepubliseer" + +#, fuzzy +#| msgid "Published From" +msgid "A Map was published" +msgstr "Gepubliseer" + +#, fuzzy +#| msgid "Map Created" +msgid "Map Deleted" +msgstr "Kaart geskep" + +#, fuzzy +#| msgid "A map was commented on" +msgid "A Map was deleted" +msgstr "'N kaart is kommentaar op" + +msgid "Comment on Map" +msgstr "Kommentaar op kaart" + +msgid "A map was commented on" +msgstr "'N kaart is kommentaar op" + +msgid "Rating for Map" +msgstr "Waardering vir kaart" + +msgid "A rating was given to a map" +msgstr "A-gradering is om 'n kaart gegee" + +msgid "Featured Map URL" +msgstr "Gewilde Map URL" + +msgid "stack order" +msgstr "stapel om" + +#, fuzzy +#| msgid "format" +msgid "format" +msgstr "formaat" + +msgid "name" +msgstr "naam" + +msgid "store" +msgstr "" + +msgid "opacity" +msgstr "ondeursigtigheid" + +msgid "styles" +msgstr "style" + +msgid "transparent" +msgstr "deursigtige" + +msgid "fixed" +msgstr "vaste" + +#, fuzzy +#| msgid "group" +msgid "group" +msgstr "groep" + +msgid "visibility" +msgstr "sigbaarheid" + +msgid "ows URL" +msgstr "kruie URL" + +msgid "layer params" +msgstr "laag params" + +msgid "source params" +msgstr "bron params" + +msgid "Map layers WMS GetCapabilities document" +msgstr "" + +msgid "Rate this Map" +msgstr "Gradeer hierdie Map" + +msgid "Download Map" +msgstr "Zwelihle" + +msgid "Map" +msgid_plural "Maps" +msgstr[0] "" +msgstr[1] "" + +msgid "Map Layers" +msgstr "Kaartlae" + +msgid "This map uses the following layers:" +msgstr "Hierdie kaart gebruik die volgende lae:" + +msgid "Documents related to this map" +msgstr "Dokumente wat verband hou met die kaart" + +msgid "List of documents related to this map:" +msgstr "Lys van dokumente wat verband hou met hierdie kaart:" + +msgid "Change Permissions of this Map" +msgstr "Verandering regte van die kaart" + +msgid "Copy this map" +msgstr "Kopieer die kaart" + +msgid "Duplicate this map and modify it for your own purposes" +msgstr "Dupliseer die kaart en verander dit vir jou eie doeleindes" + +msgid "Create a New Map" +msgstr "Skep 'n nuwe gids" + +msgid "" +"\n" +" Here you can download all the layers of this map that\n" +" are hosted on this GeoNode.\n" +" " +msgstr "" +"\n" +"Hier kan jy die aflaai van al die lae van die kaart wat aangebied \n" +"word op hierdie GeoNode." + +msgid "" +"\n" +"
          Could not find downloadable layers " +"for this map. You can go back to \n" +" " +msgstr "" +"\n" +"
          Kon dit nie vind afgelaai lae vir die " +"kaart. Jy kan terug gaan na" + +msgid "" +"\n" +" Additionally, the map contains these layers which will not be " +"downloaded\n" +" due to security restrictions:\n" +" " +msgstr "" +"\n" +"Daarbenewens het die kaart bevat hierdie lae wat nie afgelaai word nie \n" +"as gevolg van sekuriteit beperkinge:" + +msgid "" +"\n" +" Finally, the map contains these layers which will not be downloaded\n" +" because they are not available directly from this GeoNode:\n" +" " +msgstr "" +"\n" +"Ten slotte, die kaart bevat hierdie lae wat nie afgelaai word nie, \n" +"want hulle is nie beskikbaar direk van hierdie GeoNode:" + +msgid "Downloading" +msgstr "Aflaai" + +msgid "Retry" +msgstr "Weer probeer" + +msgid "Start downloading this map" +msgstr "Begin hierdie kaart aflaai" + +msgid "Download Complete" +msgstr "Aflaai Complete" + +msgid "Download Failed" +msgstr "Aflaai misluk" + +msgid "Edit Map" +msgstr "Wysig Map" + +msgid "Explore Maps" +msgstr "Verken Maps" + +msgid "" +"Note: this map's orginal metadata was populated by importing a metadata XML " +"file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" +"Let wel: die kaart se oorspronklike metadata is bevolk deur die invoer van " +"'n metadata XML-lêer. \n" +"GeoNode se metadata invoer ondersteun 'n subset van die ISO, Fgdc en Dublin " +"Core metadata elemente. \n" +"Sommige van jou oorspronklike metadata verlore gewees het." + +#, fuzzy +#| msgid "Return to" +msgid "Return to Map" +msgstr "Terug te keer na" + +msgid "Remove Map" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(map_title)s?\n" +" " +msgstr "" + +msgid "You are not permitted to delete this map." +msgstr "Jy word nie toegelaat om die kaart te verwyder." + +msgid "You do not have permissions for this map." +msgstr "Jy het nie toestemming om hierdie kaart." + +msgid "You must be logged in to save this map" +msgstr "Jy moet aangeteken word om die kaart te red" + +msgid "You are not permitted to save or edit this map." +msgstr "Jy word nie toegelaat om op te slaan of die kaart te wysig." + +msgid "You are not allowed to modify this map's metadata." +msgstr "Jy word nie toegelaat om die kaart se metadata te verander." + +msgid "You are not allowed to view this map." +msgstr "Jy word nie toegelaat om die kaart te sien." + +#, fuzzy +#| msgid "Thumbnail" +msgid "error saving thumbnail" +msgstr "Thumbnail" + +msgid "Message received" +msgstr "" + +msgid "New message received in one of your threads" +msgstr "" + +msgid "Monitoring alert" +msgstr "" + +msgid "Alert situation reported by monitoring" +msgstr "" + +msgid "GeoNode Monitoring on {base_ctx['host']} " +msgstr "" + +#, fuzzy +#| msgid "Email Address" +msgid "Enter valid email addresses." +msgstr "E-posadres" + +msgid "Show list of services" +msgstr "" + +msgid "" +"Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, last sync will be used." +msgstr "" + +msgid "" +"Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, now will be used." +msgstr "" + +msgid "Force check" +msgstr "" + +msgid "Format of audit log (xml, json)" +msgstr "" + +msgid "" +"Should old data be preserved (default: no, data older than settings." +"MONITORING_DATA_TTL will be removed)" +msgstr "" + +msgid "Should stop on first error occured (default: no)" +msgstr "" + +msgid "Should process and send notifications as well (default: no)" +msgstr "" + +msgid "Collect data from this service only" +msgstr "" + +msgid "Show list of metrics" +msgstr "" + +msgid "Show list of labels for metric" +msgstr "" + +#, fuzzy +#| msgid "List of resources using this document:" +msgid "Show list of resources for metric" +msgstr "Lys van hulpbronne die gebruik van hierdie dokument:" + +msgid "Data aggregation interval in seconds (default: 60)" +msgstr "" + +#, fuzzy +#| msgid "attribute name" +msgid "Metric name" +msgstr "kenmerk naam" + +#, fuzzy +#| msgid "reference date for the cited resource" +msgid "Show data for specific resource in resource_type=resource_name format" +msgstr "verwysing datum vir die aangehaal hulpbron" + +#, fuzzy +#| msgid "reference date for the cited resource" +msgid "Show data for specific resource" +msgstr "verwysing datum vir die aangehaal hulpbron" + +#, fuzzy +#| msgid "reference date for the cited resource" +msgid "Show data for specific label" +msgstr "verwysing datum vir die aangehaal hulpbron" + +#, python-brace-format +msgid "Write result to file, default GEOIP_PATH: {settings.GEOIP_PATH}" +msgstr "" + +msgid "" +"Fetch database from specific url. If nothing provided, default {} will be " +"used" +msgstr "" + +msgid "Overwrite file if exists" +msgstr "" + +msgid "GeoNode" +msgstr "GeoNode" + +msgid "GeoServer" +msgstr "Geoserver" + +#, fuzzy +#| msgid "GeoServer" +msgid "Host (GeoServer)" +msgstr "Geoserver" + +#, fuzzy +#| msgid "About GeoNode" +msgid "Host (GeoNode)" +msgstr "Oor GeoNode" + +#, fuzzy +#| msgid "Resources" +msgid "No resource" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "Resources" +msgid "Resource base" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "Styles" +msgid "Style" +msgstr "Style" + +msgid "Admin" +msgstr "Admin" + +msgid "Other" +msgstr "" + +msgid "Rate" +msgstr "" + +#, fuzzy +#| msgid "count" +msgid "Count" +msgstr "tel" + +msgid "Value" +msgstr "" + +msgid "Value numeric" +msgstr "" + +msgid "Bytes" +msgstr "" + +msgid "Kilobytes" +msgstr "" + +msgid "Megabytes" +msgstr "" + +msgid "Gigabytes" +msgstr "" + +msgid "Bytes per second" +msgstr "" + +msgid "Kilobytes per second" +msgstr "" + +msgid "Megabytes per second" +msgstr "" + +msgid "Gigabytes per second" +msgstr "" + +msgid "Seconds" +msgstr "" + +msgid "Percentage" +msgstr "" + +#, fuzzy +#| msgid "local OWS" +msgid "Not OWS" +msgstr "plaaslike OWS" + +msgid "Any OWS" +msgstr "" + +msgid "Change" +msgstr "Verandering" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "Change Metadata" +msgstr "Wysig Metadata" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "View Metadata" +msgstr "Wysig Metadata" + +#, fuzzy +#| msgid "Public" +msgid "Publish" +msgstr "Openbare" + +#, fuzzy +#| msgid "GeoServer" +msgid "Geoserver event" +msgstr "Geoserver" + +#, fuzzy +#| msgid "List of resources using this document:" +msgid "List of resources affected" +msgstr "Lys van hulpbronne die gebruik van hierdie dokument:" + +#, fuzzy +#| msgid "Resource using this document" +msgid "Response processing time in ms" +msgstr "Hulpbron gebruik van hierdie dokument" + +msgid "1 minute" +msgstr "" + +msgid "5 minutes" +msgstr "" + +msgid "10 minutes" +msgstr "" + +msgid "30 minutes" +msgstr "" + +msgid "1 hour" +msgstr "" + +msgid "Error" +msgstr "" + +msgid "Fatal" +msgstr "" + +msgid "Expected min/max values for user configuration" +msgstr "" + +msgid "Marker of last delivery" +msgstr "" + +msgid "Minimum time between subsequent notifications" +msgstr "" + +msgid "How severe would be error from this notification" +msgstr "" + +msgid "Is it active" +msgstr "" + +#, python-brace-format +msgid "{self.severity}: {self.name}" +msgstr "" + +msgid "Value must be above" +msgstr "" + +msgid "Value must be below" +msgstr "" + +msgid "Last update must not be older than" +msgstr "" + +msgid "Max timeout for given metric before error should be raised" +msgstr "" + +msgid "Monitoring & Analytics" +msgstr "" + +msgid "Only superusers can monitor performance" +msgstr "" + +msgid "Hi," +msgstr "" + +msgid "Monitoring on" +msgstr "" + +#, fuzzy +#| msgid "The following groups:" +msgid "has detected following problems :" +msgstr "Die volgende groepe:" + +msgid "spotted" +msgstr "" + +msgid "ago" +msgstr "gelede" + +msgid "emerged" +msgstr "" + +msgid "User following you" +msgstr "Gebruiker wat u volg" + +#, fuzzy +#| msgid "This user is not following anyone" +msgid "Another user has started following you" +msgstr "Hierdie gebruiker is nie as gevolg van iemand" + +msgid "User requested access" +msgstr "" + +msgid "A new user has requested access to the site" +msgstr "" + +#, fuzzy +#| msgid "Recent activity" +msgid "Account activated" +msgstr "Onlangse aktiwiteit" + +msgid "This account is now active and can log in the site" +msgstr "" + +msgid "Personal info" +msgstr "Personal info" + +msgid "Important dates" +msgstr "Belangrike datums" + +#, fuzzy +#| msgid "Edit profile" +msgid "Extended profile" +msgstr "Wysig profiel" + +msgid "Password changed successfully." +msgstr "Wagwoord suksesvol verander." + +#, python-format +msgid "Change password: %s" +msgstr "Verander wagwoord: %s" + +msgid "party who authored the resource" +msgstr "party wat die hulpbron outeur" + +msgid "" +"party who has processed the data in a manner such that the resource has been " +"modified" +msgstr "" +"party wat verwerk het die data op 'n wyse soos wat die hulpbron is verander" + +msgid "party who published the resource" +msgstr "party wat die hulpbron gepubliseer" + +msgid "" +"party that accepts accountability and responsibility for the data and " +"ensures appropriate care and maintenance of the resource" +msgstr "" +"party wat aanvaar aanspreeklikheid en verantwoordelikheid vir die data en " +"verseker die nodige sorg en instandhouding van die hulpbron" + +msgid "" +"party who can be contacted for acquiring knowledge about or acquisition of " +"the resource" +msgstr "" +"party wat gekontak kan word vir die verkryging van kennis oor of verkryging " +"van die hulpbron" + +msgid "party who distributes the resource" +msgstr "party wat die hulpbron versprei" + +msgid "party who uses the resource" +msgstr "party wat die hulpbron gebruik" + +msgid "party that supplies the resource" +msgstr "party wat verskaf die hulpbron" + +msgid "party who created the resource" +msgstr "party wat die hulpbron geskep" + +msgid "party that owns the resource" +msgstr "party wat die eienaar van die hulpbron" + +msgid "key party responsible for gathering information and conducting research" +msgstr "" +"sleutel party wat verantwoordelik is vir die insameling van inligting en die " +"uitvoer van navorsing" + +#, fuzzy +#| msgid "Email Address" +msgid "Email Address" +msgstr "E-posadres" + +#, fuzzy +#| msgid "Organization Name" +msgid "Organization Name" +msgstr "Organisasie se naam" + +msgid "name of the responsible organization" +msgstr "naam van die verantwoordelike organisasie" + +msgid "Profile" +msgstr "Profiel" + +msgid "introduce yourself" +msgstr "stel jouself" + +msgid "Position Name" +msgstr "Posisie Naam" + +msgid "role or position of the responsible person" +msgstr "rol of posisie van die verantwoordelike persoon" + +msgid "Voice" +msgstr "Voice" + +msgid "" +"telephone number by which individuals can speak to the responsible " +"organization or individual" +msgstr "" +"telefoonnommer waardeur individue kan praat met die verantwoordelike " +"organisasie of individu" + +msgid "Facsimile" +msgstr "Faks" + +msgid "" +"telephone number of a facsimile machine for the responsible organization or " +"individual" +msgstr "" +"telefoonnommer van 'n faks masjien vir die verantwoordelike organisasie of " +"individu" + +msgid "Delivery Point" +msgstr "Lewering punt" + +msgid "" +"physical and email address at which the organization or individual may be " +"contacted" +msgstr "" +"fisiese en e-posadres wat die organisasie of individu kan gekontak word" + +msgid "City" +msgstr "Plaas" + +msgid "city of the location" +msgstr "stad van die plek" + +msgid "Administrative Area" +msgstr "Administratiewe Area" + +msgid "state, province of the location" +msgstr "staat, provinsie van die plek" + +msgid "Postal Code" +msgstr "Poskode" + +msgid "ZIP or other postal code" +msgstr "Zip of ander poskode" + +#, fuzzy +#| msgid "count" +msgid "Country" +msgstr "tel" + +msgid "country of the physical address" +msgstr "land van die fisiese adres" + +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated" +msgstr "" +"algemeen gebruik woord (e) of geformaliseer woord (e) of frase (s) gebruik " +"word om die vak (ruimte of komma geskei te beskryf" + +msgid "Timezone" +msgstr "" + +msgid "No Group" +msgstr "Geen groep" + +msgid "All contents" +msgstr "Alle inhoud" + +#, fuzzy +#| msgid "Organization Name" +msgid "No Organization Info" +msgstr "Organisasie se naam" + +msgid "Forgot Username" +msgstr "Vergeet" + +msgid "" +"Enter your email address and click the submit button.
          Your username " +"will be sent to you." +msgstr "" + +msgid "Create Profile" +msgstr "Skep Profiel" + +msgid "Profile of " +msgstr "Profiel van" + +msgid "Not provided." +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Organization" +msgstr "" + +msgid "Not provided" +msgstr "Nie voorsien" + +msgid "Fax" +msgstr "" + +msgid "User layers WMS GetCapabilities document" +msgstr "" + +msgid "Message User" +msgstr "" + +msgid "Edit profile" +msgstr "Wysig profiel" + +msgid "Connected social accounts" +msgstr "" + +msgid "Associated e-mails" +msgstr "" + +#, fuzzy +#| msgid "Change password" +msgid "Set/Change password" +msgstr "Verandering wagwoord" + +msgid "Upload new layers" +msgstr "Oplaai nuwe lae" + +#, fuzzy +#| msgid "Create a new map" +msgid "Create a new layer" +msgstr "Skep 'n nuwe kaart" + +#, fuzzy +#| msgid "Upload Document" +msgid "Upload new document" +msgstr "Oplaai Document" + +msgid "Create a new map" +msgstr "Skep 'n nuwe kaart" + +msgid "My Activities" +msgstr "My Aktiwiteite" + +msgid "Notifications" +msgstr "" + +msgid "Announcements" +msgstr "Aankondigings" + +#, fuzzy +#| msgid "Invite User" +msgid "Invite Users" +msgstr "Nooi gebruiker" + +msgid "User Activities" +msgstr "Gebruikers Aktiwiteite" + +msgid "Resources" +msgstr "Hulpbronne" + +msgid "Edit Your Profile" +msgstr "Verander jou profiel" + +msgid "Edit Profile for" +msgstr "" + +msgid "Change your avatar" +msgstr "Verander jou avatar" + +msgid "Update profile" +msgstr "Werk profiel" + +msgid "Explore People" +msgstr "Verken Mense" + +#, fuzzy, python-brace-format +#| msgid "Your username for " +msgid "Your username for {site.name}" +msgstr "Jou gebruikersnaam vir" + +msgid "Your username has been emailed to you." +msgstr "Jou gebruikersnaam is per e-pos aan jou." + +msgid "No user could be found with that email address." +msgstr "Geen gebruiker gevind kon word met die e-pos adres." + +#, fuzzy +#| msgid "You are not permitted to save or edit this map." +msgid "You are not permitted to save or edit this resource." +msgstr "Jy word nie toegelaat om op te slaan of die kaart te wysig." + +#, fuzzy +#| msgid "You are not allowed to view this document." +msgid "You are not authorized to download this resource." +msgstr "Jy word nie toegelaat om hierdie dokument te besigtig." + +msgid "" +"No files have been found for this resource. Please, contact a system " +"administrator." +msgstr "" + +#, fuzzy +#| msgid "Documents" +msgid "No files found." +msgstr "Dokumente" + +#, fuzzy +#| msgid "Not provided" +msgid "Not Authorized" +msgstr "Nie voorsien" + +#, fuzzy +#| msgid "GeoNode Help" +msgid "GeoNode Resource Processing Library" +msgstr "GeoNode Hulp" + +msgid "Disabling this Task will make the Processing Workflow to skip it." +msgstr "" + +msgid "Processing Workflow Tasks" +msgstr "" + +msgid "Session is Expired. Please login again!" +msgstr "" + +msgid "Permissions successfully updated!" +msgstr "" + +#, fuzzy +#| msgid "Permissions" +msgid "Error updating permissions :(" +msgstr "Permissions" + +#, fuzzy +#| msgid "You do not have permissions for this layer." +msgid "You are not allowed to change permissions for this resource" +msgstr "Jy het nie toestemming om hierdie laag." + +msgid "Security Rules Cache Refreshed!" +msgstr "" + +#, fuzzy +#| msgid "You are not permitted to save or edit this map." +msgid "You cannot modify this resource!" +msgstr "Jy word nie toegelaat om op te slaan of die kaart te wysig." + +#, fuzzy +#| msgid "Password changed successfully." +msgid "Attributes/Stats Refreshed Successfully!" +msgstr "Wagwoord suksesvol verander." + +msgid "GeoWebCache Tiled Layer Emptied!" +msgstr "" + +#, fuzzy +#| msgid "Set permissions" +msgid "Wrong permissions specification" +msgstr "Stel regte" + +#, fuzzy +#| msgid "Change password: %s" +msgid "error delivering notification" +msgstr "Verander wagwoord: %s" + +msgid "Auto-detect" +msgstr "Auto-spoor" + +msgid "Paired WMS/WFS/WCS" +msgstr "Pare WMS / WFS / WCS" + +msgid "Web Map Service" +msgstr "Web Map Service" + +msgid "Catalogue Service" +msgstr "Katalogus Service" + +#, fuzzy +#| msgid "ArcGIS REST Service" +msgid "ArcGIS REST MapServer" +msgstr "ArcGIS REST Service" + +#, fuzzy +#| msgid "ArcGIS REST Service" +msgid "ArcGIS REST ImageServer" +msgstr "ArcGIS REST Service" + +msgid "OpenGeoPortal" +msgstr "OpenGeoPortal" + +msgid "Harvard Geospatial Library" +msgstr "Harvard Geospatial Biblioteek" + +#, fuzzy +#| msgid "GeoNode's Web Services" +msgid "GeoNode (Web Map Service)" +msgstr "GeoNode se Web Services" + +#, fuzzy +#| msgid "Catalogue Service" +msgid "GeoNode (Catalogue Service)" +msgstr "Katalogus Service" + +msgid "Service URL" +msgstr "Service URL" + +msgid "Service Type" +msgstr "Service Tipe" + +#, fuzzy, python-format +#| msgid "service interfaces" +msgid "Service %(url)s is already registered" +msgstr "diens koppelvlakke" + +#, python-format +msgid "Could not connect to the service at %(url)s" +msgstr "" + +#, python-format +msgid "Could not find importable resources for the service at %(url)s" +msgstr "" + +#, python-format +msgid "Found service of type %(found_type)s instead of %(service_type)s" +msgstr "" + +msgid "Local" +msgstr "Plaaslike" + +msgid "Cascaded" +msgstr "Kaskade" + +msgid "Harvested" +msgstr "Geoes" + +msgid "Indexed" +msgstr "Kruip" + +msgid "Live" +msgstr "Live" + +#, fuzzy +#| msgid "Resources" +msgid "Resource is queued" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "Remote Services" +msgid "Remote service unavailable" +msgstr "Afgeleë Services" + +msgid "Service unavailable" +msgstr "" + +#, fuzzy +#| msgid "Point of Contact" +msgid "Could not contact" +msgstr "Punt van kontak" + +#, fuzzy +#| msgid "Contact:" +msgid "Contact" +msgstr "Kontak:" + +#, fuzzy +#| msgid "Resources" +msgid "Service Resources" +msgstr "Hulpbronne" + +msgid "No resources have been imported yet." +msgstr "" + +#, fuzzy +#| msgid "Retry" +msgid "Retry job" +msgstr "Weer probeer" + +msgid "previous" +msgstr "" + +msgid "next" +msgstr "" + +msgid "Connecting to service..." +msgstr "" + +msgid "Edit Service Metadata" +msgstr "Wysig Service Metadata" + +#, fuzzy +#| msgid "Remote Services" +msgid "Import Service Resources" +msgstr "Afgeleë Services" + +msgid "Remove Service" +msgstr "Verwyder Service" + +msgid "Edit Service" +msgstr "" + +msgid "Services" +msgstr "Dienste" + +#, fuzzy +#| msgid "Remote Services" +msgid "Add Remote Service" +msgstr "Afgeleë Services" + +msgid "Remote Services" +msgstr "Afgeleë Services" + +#, fuzzy +#| msgid "service interfaces" +msgid "No services registered" +msgstr "diens koppelvlakke" + +#, fuzzy +#| msgid "Register a new Service" +msgid "Register Service" +msgstr "Registreer 'n nuwe diens" + +msgid "Register New Service" +msgstr "" + +msgid "Service has been created:" +msgstr "Diens is geskep:" + +msgid "The following layers will be imported" +msgstr "Die volgende lae ingevoer word" + +msgid "Remove Remote Service" +msgstr "" + +msgid "Are you sure you want to remove" +msgstr "Is jy seker jy wil verwyder" + +msgid "Deleting service and its associated resources..." +msgstr "" + +#, fuzzy +#| msgid "Resources" +msgid "Import resources" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "now" +msgid "no" +msgstr "nou" + +msgid "resources can be imported" +msgstr "" + +msgid "- These will be cascaded through your local geoserver instance" +msgstr "" + +msgid "Id" +msgstr "" + +#, fuzzy +#| msgid "Clear all filters" +msgid "Clear Filter" +msgstr "Clear all filters" + +msgid "Back to service details" +msgstr "" + +#, fuzzy +#| msgid "Resources" +msgid "Import Resources" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "Preview" +msgid "Previous" +msgstr "Voorskou" + +msgid "All resources have already been imported" +msgstr "" + +msgid "Harvesting resources..." +msgstr "" + +msgid "Re-scan Service for new Resources" +msgstr "" + +#, fuzzy +#| msgid "service interfaces" +msgid "Service registered successfully" +msgstr "diens koppelvlakke" + +msgid "The selected resources are being imported" +msgstr "" + +msgid "The selected resources have been imported" +msgstr "" + +msgid "Resource is already being processed" +msgstr "" + +#, fuzzy, python-brace-format +#| msgid "Resources" +msgid "Resource {resource_id} is being processed" +msgstr "Hulpbronne" + +#, fuzzy +#| msgid "Password changed successfully." +msgid "Service rescanned successfully" +msgstr "Wagwoord suksesvol verander." + +#, fuzzy +#| msgid "You are not permitted to save or edit this map." +msgid "You dont have enougth rigths to see the resource detail" +msgstr "Jy word nie toegelaat om op te slaan of die kaart te wysig." + +#, fuzzy +#| msgid "You are not permitted to remove this service." +msgid "You are not permitted to change this service." +msgstr "Jy word nie toegelaat om hierdie diens te verwyder nie." + +msgid "You are not permitted to remove this service." +msgstr "Jy word nie toegelaat om hierdie diens te verwyder nie." + +#, fuzzy, python-brace-format +#| msgid "Service has been created:" +msgid "Service {service.name} has been deleted" +msgstr "Diens is geskep:" + +msgid "created" +msgstr "geskep" + +msgid "deleted" +msgstr "geskrap" + +#, fuzzy +#| msgid "Update" +msgid "updated" +msgstr "Werk" + +msgid "added a comment" +msgstr "het 'n opmerking" + +msgid "updated a comment" +msgstr "updated 'n opmerking" + +msgid "uploaded" +msgstr "foto" + +msgid "Recent activity" +msgstr "Onlangse aktiwiteit" + +msgid "to" +msgstr "te" + +msgid "on" +msgstr "op" + +msgid "Not Permitted" +msgstr "Nie toegelaat" + +#, fuzzy +#| msgid "" +#| "\n" +#| " You are not allowed to perform this operation. \n" +#| " " +msgid "" +"\n" +" You are not allowed to perform this operation.\n" +" " +msgstr "" +"\n" +"Jy word nie toegelaat om hierdie aksie uit te voer." + +msgid "Please verify that you are logged in as the correct user." +msgstr "Maak seker dat jy aangeteken in die korrekte gebruiker." + +msgid "Please login or register and retry." +msgstr "Asseblief aanmeld of registreer en probeer weer." + +msgid "Page Not Found" +msgstr "Bladsy nie gevind" + +#, fuzzy +#| msgid "" +#| "\n" +#| " The page you requested does not exist. Perhaps you are using an " +#| "outdated bookmark?\n" +#| " " +msgid "" +"\n" +" The page you requested does not exist. Perhaps you are using an " +"outdated bookmark?\n" +" " +msgstr "" +"\n" +"Die bladsy wat jy aangevra het bestaan ​​nie. Miskien het jy gebruik 'n " +"verouderde boekmerk?" + +msgid "Toggle navigation" +msgstr "" + +#, fuzzy +#| msgid "dataset" +msgid "Datasets" +msgstr "dataset" + +msgid "People" +msgstr "Mense" + +msgid "Search" +msgstr "Soek" + +msgid "Register" +msgstr "Register" + +msgid "Sign in" +msgstr "Teken in" + +msgid "You are using an outdated browser that is not supported by GeoNode." +msgstr "" + +msgid "" +"Please use a modern browser like Mozilla Firefox, Google " +"Chrome or Safari." +msgstr "" + +msgid "There was a problem loading this page" +msgstr "Daar was 'n probleem met die laai hierdie bladsy" + +msgid "" +"\n" +" Please contact your GeoNode administrator (they may have " +"received an email automatically if they configured it properly).\n" +" If you are the site administrator, enable debug mode to see the " +"actual error and fix it or file an issue in GeoNode's issue tracker\n" +" " +msgstr "" + +#, fuzzy +#| msgid "Create a new map" +msgid "Create a new dataset" +msgstr "Skep 'n nuwe kaart" + +#, fuzzy +#| msgid "Create a Map" +msgid "Create Map" +msgstr "Skep 'n kaart" + +#, fuzzy +#| msgid "Add Layers" +msgid "Add User" +msgstr "Voeg Lae" + +msgid "Powered by" +msgstr "Aangedryf deur" + +msgid "version" +msgstr "weergawe" + +msgid "Developers" +msgstr "Ontwikkelaars" + +msgid "Username" +msgstr "Gebruikersnaam" + +msgid "Password" +msgstr "Wagwoord" + +msgid "Remember Me" +msgstr "Onthou my" + +msgid "Menu" +msgstr "" + +msgid "Recent Activity" +msgstr "Onlangse aktiwiteite" + +msgid "Inbox" +msgstr "Boks" + +msgid "Help" +msgstr "Help" + +msgid "Log out" +msgstr "Teken uit" + +msgid "Info" +msgstr "Info" + +msgid "Granules" +msgstr "" + +msgid "Share" +msgstr "Deel" + +msgid "Ratings" +msgstr "Graderings" + +msgid "Exif" +msgstr "" + +msgid "Set permissions on selected resources" +msgstr "Stel regte op geselekteerde bronne" + +msgid "Apply Changes" +msgstr "Pas veranderinge" + +msgid "total" +msgstr "" + +msgid "Add Comment" +msgstr "Voeg Kommentaar" + +msgid "Log in to add a comment" +msgstr "Teken in 'n kommentaar by te voeg" + +msgid "By" +msgstr "Deur" + +msgid "Submit Comment" +msgstr "" + +msgid "Who can view it?" +msgstr "Wie dit kan sien?" + +msgid "Anyone" +msgstr "Enigiemand" + +msgid "The following users:" +msgstr "Die volgende gebruikers:" + +msgid "The following groups:" +msgstr "Die volgende groepe:" + +msgid "Who can download it?" +msgstr "Wie kan dit aflaai?" + +msgid "Who can change metadata for it?" +msgstr "Wie kan metadata vir dit verander?" + +#, fuzzy +#| msgid "Who can edit data for this layer?" +msgid "Who can edit data for this dataset?" +msgstr "Wie data vir hierdie laag kan verander?" + +#, fuzzy +#| msgid "Who can edit styles for this layer?" +msgid "Who can edit styles for this dataset?" +msgstr "Wie kan style vir hierdie laag wysig?" + +msgid "" +"Who can manage it? (update, delete, change permissions, publish/unpublish it)" +msgstr "" +"Wie kan dit bestuur? (Update, verwyder, verandering regte, publiseer / " +"ongedaan dit)" + +msgid "Set permissions for this resource" +msgstr "Stel regte vir hierdie hulpbron" + +#, fuzzy +#| msgid "Explore Groups" +msgid "People and Groups" +msgstr "Verken Groepe" + +msgid "Geo Limits" +msgstr "" + +#, fuzzy +#| msgid "Add Layers" +msgid "Users" +msgstr "Voeg Lae" + +msgid "Show Geo Limits" +msgstr "" + +#, fuzzy +#| msgid "Upload Documents" +msgid "Upload Geo Limits" +msgstr "Oplaai dokumente" + +msgid "Save Geo Limits" +msgstr "" + +#, fuzzy +#| msgid "Delete These" +msgid "Delete Geo Limits" +msgstr "Vee hierdie" + +#, fuzzy +#| msgid "Message Inbox" +msgid "Message box" +msgstr "Boodskap Inbox" + +msgid "OK" +msgstr "" + +msgid "Mandatory files : SHP , DBF" +msgstr "" + +msgid "" +"Upload a ZIP file containing an ESRI Shapefile. If the ZIP provides also a ." +"prj file, you don't have to specify the EPSG SRID" +msgstr "" + +#, fuzzy +#| msgid "Choose users..." +msgid "Choose" +msgstr "Kies gebruikers ..." + +#, fuzzy +#| msgid "Choose users..." +msgid "Choose a file..." +msgstr "Kies gebruikers ..." + +msgid "Reset" +msgstr "" + +msgid "Only the SRID number without the 'EPSG:' prefix!" +msgstr "" + +msgid "Default : 4326" +msgstr "" + +msgid "Set your desired encoding UTF-8, Big5, Big5-HKSCS ..." +msgstr "" + +msgid "Load SHP-ZIP" +msgstr "" + +msgid "" +"

          This will remove the Geo Limits currently drawn on the map.

          In " +"order to store the Geo Limits you will need to save them " +"anyway.

          Do you want to proceed?

          " +msgstr "" + +msgid "" +"

          This will override the current stored Geo Limits on the DB.

          To " +"apply them you will need to click on Apply Changes button " +"anyway.

          WARNING: This operation cannot be reverted!

          Do you want to proceed?

          " +msgstr "" + +msgid "Save Geo Limit" +msgstr "" + +msgid "" +"

          Geometry successfully saved!

          To apply them you " +"will need to click on Apply Changes button.

          " +msgstr "" + +msgid "

          Error while trying to save the Geometry!

          " +msgstr "" + +msgid "Load Geo Limits" +msgstr "" + +msgid "Load Geo Limit" +msgstr "" + +msgid "

          No Geometry found!

          " +msgstr "" + +msgid "" +"

          This will permanently remove the Geo Limits from the DB." +"

          To apply them you will need to click on Apply Changes button.

          Do you want to proceed?

          " +msgstr "" + +msgid "Delete Geo Limit" +msgstr "" + +msgid "

          Geo Limitsy successfully deleted!

          " +msgstr "" + +msgid "" +"

          Error occurred while trying to delete Geo Limits:

          " +msgstr "" + +msgid "Choose users..." +msgstr "Kies gebruikers ..." + +msgid "Choose groups..." +msgstr "Kies groepe ..." + +msgid "Placeholder for status-message" +msgstr "" + +msgid "Placeholder for status-message-body" +msgstr "" + +msgid "About GeoNode" +msgstr "Oor GeoNode" + +msgid "" +"GeoNode is a geospatial content management system, a platform for the " +"management and publication of geospatial data. It brings together mature and " +"stable open-source software projects under a consistent and easy-to-use " +"interface allowing non-specialized users to share data and create " +"interactive maps." +msgstr "" + +msgid "" +"Data management tools built into GeoNode allow for integrated creation of " +"data, metadata, and map visualizations. Each dataset in the system can be " +"shared publicly or restricted to allow access to only specific users. Social " +"features like user profiles and commenting and rating systems allow for the " +"development of communities around each platform to facilitate the use, " +"management, and quality control of the data the GeoNode instance contains." +msgstr "" + +msgid "" +"It is also designed to be a flexible platform that software developers can " +"extend, modify or integrate against to meet requirements in their own " +"applications." +msgstr "" + +msgid "Account Inactive" +msgstr "" + +msgid "This account is inactive." +msgstr "" + +msgid "Account Pending Approval" +msgstr "" + +#, python-format +msgid "" +"We have sent the administrators a notice to approve your account associated " +"with %(email)s. If the account is approved, you will receive a " +"confirmation notice." +msgstr "" + +#, fuzzy +#| msgid "count" +msgid "Account" +msgstr "tel" + +#, fuzzy +#| msgid "Email Address" +msgid "E-mail Addresses" +msgstr "E-posadres" + +msgid "The following e-mail addresses are associated with your account:" +msgstr "" + +msgid "Verified" +msgstr "" + +msgid "Unverified" +msgstr "" + +#, fuzzy +#| msgid "Primary email" +msgid "Primary" +msgstr "Primary email" + +msgid "Make Primary" +msgstr "" + +#, fuzzy +#| msgid "Restrictions" +msgid "Re-send Verification" +msgstr "Beperkings" + +msgid "Warning:" +msgstr "" + +msgid "" +"You currently do not have any e-mail address set up. You should really add " +"an e-mail address so you can receive notifications, reset your password, etc." +msgstr "" + +#, fuzzy +#| msgid "Email Address" +msgid "Add E-mail Address" +msgstr "E-posadres" + +msgid "Add E-mail" +msgstr "" + +msgid "Do you really want to remove the selected e-mail address?" +msgstr "" + +#, python-format +msgid "" +"Hello from %(site_name)s!\n" +"\n" +"You're receiving this e-mail because user %(user_display)s has given yours " +"as an e-mail address to connect their account.\n" +"\n" +"To confirm this is correct, go to %(activate_url)s\n" +msgstr "" + +#, python-format +msgid "" +"Thank you from %(site_name)s!\n" +"%(site_domain)s" +msgstr "" + +#, fuzzy, python-format +#| msgid "Confirm email address for %(site_name)s" +msgid "Please confirm email address for %(site_name)s" +msgstr "Bevestig e-posadres vir %(site_name)s" + +#, python-format +msgid "You have been invited to sign up at %(site_name)s." +msgstr "" + +#, fuzzy, python-format +#| msgid "Confirm email address for %(site_name)s" +msgid "Create an account on %(site_name)s" +msgstr "Bevestig e-posadres vir %(site_name)s" + +msgid "" +"This is the email notification to confirm your password has been changed on" +msgstr "" + +#, fuzzy +#| msgid "Change password: %s" +msgid "Change password email notification" +msgstr "Verander wagwoord: %s" + +#, fuzzy, python-format +#| msgid "" +#| "You're receiving this email because you or someone else has requested a " +#| "password for your user account at %(site_name)s.\n" +#| "It can be safely ignored if you did not request a password reset. Click " +#| "the link below to reset your password.\n" +#| "\n" +#| "%(password_reset_url)s\n" +msgid "" +"Hello from %(site_name)s!\n" +"\n" +"You're receiving this e-mail because you or someone else has requested a " +"password for your user account.\n" +"It can be safely ignored if you did not request a password reset. Click the " +"link below to reset your password." +msgstr "" +"Jy hierdie e-pos ontvang omdat jy of iemand anders het 'n wagwoord aangevra " +"vir jou rekening op %(site_name)s. \n" +"Dit kan veilig geïgnoreer word as jy nie 'n wagwoord in te stel versoek het. " +"Klik op die onderstaande skakel om jou wagwoord te herstel. \n" +"\n" +"%(password_reset_url)s\n" + +#, python-format +msgid "In case you forgot, your username is %(username)s." +msgstr "" + +#, python-format +msgid "" +"Thank you for using %(site_name)s!\n" +"%(site_domain)s" +msgstr "" + +#, python-format +msgid "[%(site_name)s] Password reset" +msgstr "[%(site_name)s] Wagwoordterugstelling" + +#, fuzzy +#| msgid "Email Address" +msgid "Confirm E-mail Address" +msgstr "E-posadres" + +#, python-format +msgid "" +"Please confirm that %(email)s is an e-mail " +"address for user %(user_display)s." +msgstr "" + +msgid "Confirm" +msgstr "Bevestig" + +#, python-format +msgid "" +"This e-mail confirmation link expired or is invalid. Please issue a new e-mail confirmation request." +msgstr "" + +msgid "Log in" +msgstr "Teken in" + +msgid "Log in to an existing account" +msgstr "Teken in om 'n bestaande rekening" + +#, python-format +msgid "" +"Please sign in with one\n" +" of your existing third party accounts. Or, sign up\n" +" for a %(site_name)s account and sign in below:" +msgstr "" + +msgid "or" +msgstr "" + +#, python-format +msgid "" +"If you have not created an account yet, then please\n" +" sign up first." +msgstr "" + +#, fuzzy +#| msgid "Forgot your password?" +msgid "Forgot Password?" +msgstr "Jy jou wagwoord vergeet?" + +#, fuzzy +#| msgid "Sign in" +msgid "Sign In" +msgstr "Teken in" + +msgid "Are you sure you want to log out?" +msgstr "Is jy seker jy wil hê om aan te meld nie?" + +msgid "Password change" +msgstr "" + +msgid "Password Change" +msgstr "" + +msgid "Change your password here" +msgstr "" + +msgid "Change my password" +msgstr "" + +msgid "Password reset" +msgstr "Wagwoord herstel" + +msgid "Password Reset" +msgstr "" + +msgid "Forgotten your password?" +msgstr "" + +msgid "" +"Enter your email address below, and we'll send you an email allowing you to " +"reset it." +msgstr "" + +msgid "Reset my password" +msgstr "Weer my wagwoord" + +#, python-format +msgid "" +"If you have any trouble resetting your password, contact us at %(THEME_ACCOUNT_CONTACT_EMAIL)s." +msgstr "" +"As jy enige probleme om jou wagwoord, kontak ons by %(THEME_ACCOUNT_CONTACT_EMAIL)s." + +msgid "" +"We have sent you an e-mail. Please contact us if you do not receive it " +"within a few minutes." +msgstr "" + +#, fuzzy +#| msgid "Change password" +msgid "Change Password" +msgstr "Verandering wagwoord" + +msgid "Bad Token" +msgstr "" + +#, python-format +msgid "" +"The password reset link was invalid, possibly because it has already been " +"used. Please request a new password reset." +msgstr "" + +#, fuzzy +#| msgid "Change password" +msgid "change password" +msgstr "Verandering wagwoord" + +msgid "Your password is now changed." +msgstr "" + +#, fuzzy +#| msgid "Password" +msgid "Set Password" +msgstr "Wagwoord" + +#, fuzzy +#| msgid "Password reset" +msgid "Password Set" +msgstr "Wagwoord herstel" + +#, fuzzy +#| msgid "Forgot your password?" +msgid "Set your password here" +msgstr "Jy jou wagwoord vergeet?" + +#, fuzzy +#| msgid "Reset my password" +msgid "Set password" +msgstr "Weer my wagwoord" + +msgid "Sign up" +msgstr "Sluit aan" + +msgid "" +"Sign up with one\n" +" of your existing third party accounts" +msgstr "" + +#, fuzzy +#| msgid "Create a new map" +msgid "Create a new local account" +msgstr "Skep 'n nuwe kaart" + +msgid "Sign Up Closed" +msgstr "" + +msgid "We are sorry, but the sign up is currently closed." +msgstr "" + +msgid "Note" +msgstr "Note" + +#, python-format +msgid "you are already logged in as %(user_display)s." +msgstr "" + +msgid "Verify your email address" +msgstr "" + +msgid "" +"We have sent an email to you for verification. Follow the\n" +" link provided to finalize the signup process. Please contact us if you " +"do\n" +" not receive it within a few minutes" +msgstr "" + +#, fuzzy +#| msgid "Email Address" +msgid "Verify Your E-mail Address" +msgstr "E-posadres" + +msgid "" +"This part of the site requires us to verify that\n" +"you are who you claim to be. For this purpose, we require that you\n" +"verify ownership of your e-mail address. " +msgstr "" + +msgid "" +"We have sent an e-mail to you for\n" +"verification. Please click on the link inside this e-mail. Please\n" +"contact us if you do not receive it within a few minutes." +msgstr "" + +#, python-format +msgid "" +"Note: you can still change your e-" +"mail address." +msgstr "" + +msgid "My Activity feed" +msgstr "My aktiwiteit voer" + +msgid "Activity Feed" +msgstr "Aktiwiteit voer" + +#, fuzzy +#| msgid "Are you sure you want to remove" +msgid "Are you sure you want to run the selected" +msgstr "Is jy seker jy wil verwyder" + +#, fuzzy +#| msgid "Yes, I am sure" +msgid "Yes, I'm sure" +msgstr "Ja, ek is seker" + +msgid "GeoNode site admin" +msgstr "GeoNode administrateur" + +msgid "GeoNode administration" +msgstr "GeoNode administrasie" + +msgid "Delete Announcement?" +msgstr "" + +msgid "Are you sure you want to delete this announcement?" +msgstr "" + +msgid "Edit Announcement" +msgstr "Wysig Aankondiging" + +msgid "Create Announcement" +msgstr "Skep Aankondiging" + +msgid "New Announcement" +msgstr "Nuwe Aankondiging" + +msgid "Level" +msgstr "" + +msgid "Announcement" +msgstr "Aankondiging" + +msgid "Published From" +msgstr "Gepubliseer" + +#, python-format +msgid "" +"\n" +" Published from %(publish_start)s to %(publish_end)s.\n" +" " +msgstr "" + +#, python-format +msgid "Announcements : %(announcement)s" +msgstr "Aankondigings: %(announcement)s" + +msgid "Back to edit your profile information" +msgstr "Terug jou profiel inligting te wysig" + +msgid "Your current avatar: " +msgstr "Jou huidige avatar:" + +msgid "You haven't uploaded an avatar yet. Please upload one now." +msgstr "Jy het nie 'n avatar opgelaai nie. Laai asseblief een nou." + +msgid "Upload New Image" +msgstr "Laai New Image" + +msgid "Choose new Default" +msgstr "Kies nuwe Standaard" + +msgid "Delete Your Avatar" +msgstr "" + +msgid "Please select the avatars that you would like to delete." +msgstr "Kies die avatars wat jy wil verwyder." + +#, python-format +msgid "" +"You have no avatars to delete. Please upload one now." +msgstr "" +"Jy het geen avatars te verwyder. Let laai " +"een nou." + +msgid "Delete These" +msgstr "Vee hierdie" + +#, fuzzy +#| msgid "GeoNode Help" +msgid "GeoNode Search" +msgstr "GeoNode Hulp" + +#, fuzzy +#| msgid "Contact:" +msgid "Contact Us" +msgstr "Kontak:" + +msgid "Tel" +msgstr "" + +#, fuzzy +#| msgid "uploaded" +msgid "Uploading..." +msgstr "foto" + +#, fuzzy +#| msgid "Upload files" +msgid "Upload in progress..." +msgstr "Lêers oplaai" + +msgid "Processing..." +msgstr "" + +#, fuzzy +#| msgid "Thumbnail" +msgid "Updating Thumbnail..." +msgstr "Thumbnail" + +#, fuzzy +#| msgid "Are you sure you want to remove" +msgid "Message. Do you want to proceed?" +msgstr "Is jy seker jy wil verwyder" + +#, fuzzy +#| msgid "Resources" +msgid "Processing Resource..." +msgstr "Hulpbronne" + +msgid "Information for Developers" +msgstr "Inligting vir Developers" + +msgid "Useful information for developers interested in GeoNode." +msgstr "Nuttige inligting vir ontwikkelaars wat belangstel in GeoNode." + +#, python-format +msgid "" +"\n" +"

          GeoNode is an open service built " +"on open source software. We encourage you to build new applications using " +"the components and resources it provides. This page is a starting point for " +"developers interesting in taking full advantage of GeoNode. It also includes " +"links to the project's source code so anyone can build and customize their " +"own GeoNode.

          \n" +"\n" +"

          GeoNode Software

          \n" +"\n" +"

          All the code that runs GeoNode is open source. The code is available " +"at http://github.com/GeoNode/" +"geonode/. The issue tracker for the project is at http://github.com/GeoNode/geonode/" +"issues.

          \n" +"\n" +"

          GeoNode is built using several open source projects, each with its " +"own community. If you are interested in contributing new features to the " +"GeoNode, we encourage you to do so by contributing to one of the projects on " +"which it is built:

          \n" +"
            \n" +"
          • GeoServer - Standards based " +"server for geospatial information
            • \n" +"
          • GeoWebCache - Cache engine " +"for WMS Tiles
          • OpenLayers - " +"Pure JavaScript library powering the maps of GeoExt
            • \n" +"
          • pycsw - CSW, OpenSearch and OAI-" +"PMH metadata catalogue server
            • \n" +"
          \n" +"\n" +"

          What are OGC Services?

          \n" +"

          The data in this application is served using open standards endorsed " +"by ISO and the Open Geospatial " +"Consortium; in particular, WMS (Web Map Service) is used for accessing " +"maps, WFS (Web Feature Service) is used for accessing vector data, and WCS " +"(Web Coverage Service) is used for accessing raster data. WMC (Web Map " +"Context Documents) is used for sharing maps. You can use these services in " +"your own applications using libraries such as OpenLayers, GeoTools, and OGR " +"(all of which are open-source software and available at zero cost). " +"Additionally, CSW (Catalog Service for the Web) supports access to " +"collections of descriptive information (metadata) about data and services.\n" +"\n" +"

          What is GeoWebCache?

          \n" +"

          GeoWebCache provides mapping tiles that are compatible with a number " +"of mapping engines, including Google Maps, Bing Maps and OpenLayers. All the " +"data hosted by GeoNode is also available through GeoWebCache. GeoWebCache " +"improves on WMS by caching data and providing more responsive maps.

          \n" +"\n" +"

          CSW Example Code

          \n" +"

          To interact with GeoNode's CSW you can use any CSW client (QGIS " +"MetaSearch, GRASS, etc.). The following example illustrates a simple " +"invocation using the OWSLib Python package:

          \n" +"

          from owslib.csw import CatalogueServiceWeb

          \n" +"

          from owslib.fes import PropertyIsLike

          \n" +"

          csw = CatalogueServiceWeb('%(CATALOGUE_BASE_URL)s')

          \n" +"

          anytext = PropertyIsLike('csw:AnyText', 'birds')')

          \n" +"

          csw.getrecords2(constraints=[anytext])

          \n" +"

          print csw.results

          \n" +"

          print csw.records

          \n" +"\n" +"

          OpenLayers Example Code

          \n" +"\n" +"

          To include a GeoNode map layer in an OpenLayers map, first find the " +"name for that layer. This is found in the layer's name field " +"(not title) of the layer list. For this example, we will use " +"the Nicaraguan political boundaries background layer, whose name is " +"risk:nicaragua_admin. Then, create an instance of OpenLayers." +"Layer.WMS:

          \n" +"

          var geonodeLayer = new OpenLayers.Layer.WMS(\"GeoNode Risk Data" +"\", \"http://demo.geonode.org/geoserver/wms\",{ layers: \"risk:" +"nicaragua_admin\" });

          \n" +"\n" +"

          Google Maps Example Code

          \n" +"

          To include a GeoNode map layer in a Google Map, include the dataset " +"name in the URL template.

          \n" +"

          var tilelayer = new GTileLayer(null, null, null, " +"{tileUrlTemplate: 'http://demo.geonode.org/geoserver/gwc/service/gmaps?" +"layers=risk:nicaragua_admin&zoom={Z}&x={X}&y={Y}', isPng:true, " +"opacity:0.5 } );

          \n" +"\n" +"

          Shapefile/GeoJSON/GML Output

          \n" +"

          To get data from the GeoNode web services use the WFS protocol. For " +"example, to get the full Nicaraguan admin boundaries use:

          \n" +"

          http://demo.geonode.org/geoserver/wfs?request=GetFeature&" +"typeName=risk:nicaragua_admin&outputformat=SHAPE-ZIP

          \n" +"

          Changing output format to json, GML2, " +"GML3, or csv will get data in those formats. The " +"WFS protocol also can handle more precise queries, specifying a bounding box " +"or various spatial and non-spatial filters based on the attributes of the " +"data.

          \n" +"\n" +"

          GeoTools Example Code

          \n" +"

          Create a DataStore and extract a FeatureType from it, then run a " +"Query. It is all documented on the wiki at http://geotools.org/." +"

          \n" +" " +msgstr "" + +msgid "GeoNode's Web Services" +msgstr "GeoNode se Web Services" + +msgid "GeoNode's Web Services are available from the following URLs:" +msgstr "GeoNode se Web Services is beskikbaar by die volgende URL:" + +msgid "Dynamic tiles via WMS:" +msgstr "Dinamiese teëls via WMS:" + +msgid "Vector data via WFS:" +msgstr "Vektor data via WFS:" + +msgid "Raster data via WCS:" +msgstr "Raster data via WCS:" + +msgid "Metadata search via CSW:" +msgstr "Metadata soek via CSW:" + +msgid "Metadata search via OpenSearch:" +msgstr "" + +msgid "Metadata search via OAI-PMH:" +msgstr "" + +msgid "Cached tiles via WMTS:" +msgstr "Cached teëls via WMTS:" + +msgid "User Guide" +msgstr "User Guide" + +msgid "GeoNode Help" +msgstr "GeoNode Hulp" + +msgid "" +"This page provides helpful information about how to use the GeoNode. You can " +"use the sidebar links to navigate to what you want to know." +msgstr "" +"Hierdie bladsy bied nuttige inligting oor hoe die GeoNode te gebruik. Jy kan " +"die sidebar skakels gebruik om na wat jy wil weet." + +msgid "" +"\n" +"

          The GeoNode provides access to data sets and a map editing " +"application allows users to browse existing maps and contribute their own.\n" +"\n" +"

          Browsing Layers

          \n" +"

          The Layers tab allows you to browse data " +"uploaded to this GeoNode.

          \n" +"

          All data can be downloaded in a variety of formats, for use in other " +"applications.

          \n" +"\n" +"

          Developer Access

          \n" +"

          The Developer page is the place for " +"developers to get started building applications against the GeoNode. It " +"includes instructions on using the web services, links to the source code of " +"the GeoNode, and information about the open source projects used to create " +"it.

          \n" +"\n" +"

          Browsing Maps

          \n" +"

          The GeoNode allows users to create and share maps with one another.\n" +"

          The Maps tab is a gateway to map exploration on " +"GeoNode. From here you can search for a map or " +"create a map, which will open the Map " +"Composer.

          \n" +"
          Google Earth Mode
          \n" +"

          Any map viewed in the interactive map editor can be seen in 3D mode " +"with the Google Earth plugin. To switch to 3D mode select the Google Earth " +"globe logo, the rightmost button on top toolbar. If you do not have the " +"Google Earth plugin installed you will be prompted to install it.

          \n" +"\n" +"

          Creating a Map

          \n" +"

          To create a new map go to the Contributed Maps " +"tab and click the create your own map link.

          \n" +"

          This will take you to the Map Composer with " +"a base layer loaded.

          \n" +"

          To add data layers from the GeoNode click on the green plus button " +"located below the layers tab on the left hand side of the screen. This will " +"open a dialog listing all the layers available on the GeoNode.

          \n" +"

          To add layers to your map select them and hit the Add Layers button. When finished you may hit Done to close the " +"dialog and go back to the map.

          \n" +"\n" +"
          Reordering and Removing Layers
          \n" +"

          Change the display order of the layers listed in the data tab by " +"simply dragging and dropping their names. The order in the map will be " +"updated to reflect that. To turn a layer's visibility off simply uncheck it, " +"and to remove it entirely select it and hit the red minus button.

          \n" +"\n" +"
          Saving your map
          \n" +"

          Once a suitable set of layers and zoom level has been found it's time " +"to save it so others can see it. Click the Save button--the left most icon " +"on the top toolbar, an image of a map with a disk--on the top menu and fill " +"out the title and abstract of the map.

          \n" +"\n" +"

          Publishing a Map

          \n" +"

          Any map from the GeoNode can be embedded for use in another site or " +"blog. To publish a map:

          \n" +"
            \n" +"
          1. Select the from the list of maps on the community or map page and " +"then hit the 'Publish map' button.
            2. \n" +"
          2. Choose your desired height and width for the widget in the wizard." +"
            3. \n" +"
          3. Copy the HTML snippet provided in the wizard to any HTML page or " +"iFrame-supporting blog post.
            4. \n" +"
          \n" +"

          This will put an interactive widget showing you map in your web page " +"or blog post.

          \n" +"

          Note that the Map Composer also has a " +"button to publish the map. Just be sure to save the map before publishing if " +"there are changes that you want others to see. It publishes the last saved " +"version, not the last viewed version.

          \n" +"\n" +"

          Remixing a Map

          \n" +"

          Any map available can serve as a starting point for a new map.

          \n" +"
            \n" +"
          1. Open the map in the Map Composer.
            2. \n" +"
          2. Add and remove layers as you like.
            3. \n" +"
          3. Pan and zoom to highlight the area of interest.
            4. \n" +"
          4. IMPORTANT: Update the title, abstract, contact " +"info and tags to reflect the new map.
            5. \n" +"
          5. Save the map.
            6. \n" +"
          \n" +"

          You will be able to see your new map when you search for it from the " +"Maps tab.

          \n" +" " +msgstr "" + +msgid "Sections" +msgstr "Artikels" + +msgid "Browsing Layers" +msgstr "Blaai Lae" + +msgid "Developer Access" +msgstr "Ontwikkelaar Toegang" + +msgid "Browsing Maps" +msgstr "Blaai Maps" + +msgid "Creating a Map" +msgstr "Skep van 'n kaart" + +msgid "Exporting a Map" +msgstr "Uitvoer van 'n kaart" + +msgid "Remixing a Map" +msgstr "DJ 'n kaart" + +msgid "Welcome!" +msgstr "Welkom!" + +#, fuzzy +#| msgid "Getting Started" +msgid "Get Started »" +msgstr "Aan die slag" + +#, fuzzy +#| msgid "Search" +msgid "Search for Data." +msgstr "Soek" + +msgid "Advanced Search" +msgstr "" + +msgid "" +"Click to search for geospatial data published by other users, organizations " +"and public sources. Download data in standard formats." +msgstr "" +"Klik om te soek vir geospatiale data wat gepubliseer is deur ander " +"gebruikers, organisasies en openbare bronne. Aflaai data in die standaard " +"formaat." + +#, fuzzy +#| msgid "dataset" +msgid "Add datasets" +msgstr "dataset" + +#, fuzzy +#| msgid "Explore Layers" +msgid "Explore datasetss" +msgstr "Vind Lae" + +#, fuzzy +#| msgid "" +#| "Data is available for browsing, aggregating and styling to generate maps " +#| "which can be shared publicly or restricted to specific users only." +msgid "" +"Data is available for browsing, aggregating and styling to generate maps " +"which can be saved, downloaded, shared publicly or restricted to specify " +"users only." +msgstr "" +"Data is beskikbaar vir blaai, saamgevoeg en stilering kaarte wat publiek " +"gedeel word of beperk tot spesifieke gebruikers op te wek." + +msgid "Create maps" +msgstr "Skep kaarte" + +msgid "Explore maps" +msgstr "" + +msgid "" +"As for the layers and maps GeoNode allows to publish tabular and text data, " +"manage their metadata and associated documents." +msgstr "" + +#, fuzzy +#| msgid "Add Comment" +msgid "Add documents" +msgstr "Voeg Kommentaar" + +#, fuzzy +#| msgid "Explore Documents" +msgid "Explore documents" +msgstr "Verken dokumente" + +#, fuzzy +#| msgid "" +#| "GeoNode allows registered users to easily upload geospatial data in " +#| "several formats including shapefile and GeoTiff." +msgid "" +"Geonode allows registered users to easily upload geospatial data and various " +"documents in several formats." +msgstr "" +"GeoNode laat geregistreerde gebruikers toe om maklik te laai geospatiale " +"data in verskeie formate, insluitend shape file en Geotiff." + +msgid "See users" +msgstr "" + +#, fuzzy +#| msgid "feature type" +msgid "Featured Datasets" +msgstr "funksie tipe" + +#, fuzzy +#| msgid "Accepted" +msgid "Accept" +msgstr "Aanvaar" + +#, fuzzy +#| msgid "Subject" +msgid "Reject" +msgstr "Onderwerp" + +msgid "Leave" +msgstr "" + +msgid "Dear Sir/Madam," +msgstr "" + +#, python-format +msgid "is inviting you to join (%(site_name)s)." +msgstr "" + +#, python-format +msgid "" +"To do so, please register at %(site_name)s " +"Registration." +msgstr "" + +msgid "" +"Once you receive the confirmation that your account is activated, you can " +"notify" +msgstr "" + +msgid "that you wish to join her/his group(s) through" +msgstr "" + +msgid "this link" +msgstr "" + +#, python-format +msgid "" +"%(inviter_name)s is a member of the following group(s):" +msgstr "" + +msgid "We look forward to seeing you on the platform," +msgstr "" + +#, fuzzy +#| msgid "GeoNode site admin" +msgid "The GeoNode team." +msgstr "GeoNode administrateur" + +msgid "is inviting you to join" +msgstr "" + +msgid "To do so, please register at" +msgstr "" + +msgid "that you wish to join her/his group(s) through this link" +msgstr "" + +#, fuzzy +#| msgid "The following groups:" +msgid "is a member of the following group(s)" +msgstr "Die volgende groepe:" + +#, fuzzy +#| msgid "Download Metadata" +msgid "Updating Metadata..." +msgstr "Aflaai Metadata" + +msgid "Topic Category is mandatory and cannot be empty!" +msgstr "" + +msgid "Group is mandatory and cannot be empty!" +msgstr "" + +msgid "Identification" +msgstr "" + +msgid "Yes" +msgstr "" + +msgid "Published" +msgstr "" + +msgid "email" +msgstr "e-pos" + +#, fuzzy +#| msgid "format" +msgid "Information" +msgstr "formaat" + +msgid "Identification Image" +msgstr "" + +#, fuzzy +#| msgid "Temporal Extent" +msgid "Spatial Extent" +msgstr "Tydelike Mate" + +#, fuzzy +#| msgid "projection" +msgid "Projection System" +msgstr "projeksie" + +msgid "Extension x0" +msgstr "" + +msgid "Extension x1" +msgstr "" + +msgid "Extension y0" +msgstr "" + +msgid "Extension y1" +msgstr "" + +msgid "Features" +msgstr "" + +msgid "Contact Points" +msgstr "" + +msgid "References" +msgstr "" + +msgid "Link Online" +msgstr "" + +msgid "Metadata Page" +msgstr "" + +msgid "Online Link" +msgstr "" + +msgid "* Field declared Mandatory by the Metadata Schema" +msgstr "" + +msgid "Please, fill the following items before clicking on preview:" +msgstr "" + +#, fuzzy +#| msgid "Set permissions" +msgid "Request perission" +msgstr "Stel regte" + +#, fuzzy +#| msgid "party that owns the resource" +msgid "Request editing for resource: " +msgstr "party wat die eienaar van die hulpbron" + +#, fuzzy +#| msgid "Resources" +msgid "Back to resource" +msgstr "Hulpbronne" + +msgid "Send request" +msgstr "" + +msgid "Your account has been approved and is now active." +msgstr "" + +msgid "You can use the login form at" +msgstr "" + +#, fuzzy +#| msgid "Welcome" +msgid "Welcome at" +msgstr "Welkom" + +#, fuzzy +#| msgid "Recent activity" +msgid "Your account is now active" +msgstr "Onlangse aktiwiteit" + +msgid "has requested access to the site." +msgstr "" + +msgid "" +"You can enable access by setting the user as active on the admin section" +msgstr "" + +msgid "A user has requested access to the site" +msgstr "" + +#, fuzzy +#| msgid "The following users:" +msgid "The following document was approved" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "Powered by" +msgid "owned by" +msgstr "Aangedryf deur" + +msgid "You can visit the document's detail page here" +msgstr "" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A document has been approved" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "You do not have permissions for this document." +msgid "A comment has been posted for the document" +msgstr "Jy het nie toestemming om hierdie dokument." + +#, fuzzy +#| msgid "Choose users..." +msgid "by the user" +msgstr "Kies gebruikers ..." + +#, fuzzy +#| msgid "You do not have permissions for this document." +msgid "A comment has been posted for a document" +msgstr "Jy het nie toestemming om hierdie dokument." + +#, fuzzy +#| msgid "Choose users..." +msgid "The user" +msgstr "Kies gebruikers ..." + +#, fuzzy +#| msgid "The following users:" +msgid "uploaded the following document" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A document has been uploaded" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "The following users:" +msgid "The following document was deleted" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A document has been deleted" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "The following groups:" +msgid "The following document was published" +msgstr "Die volgende groepe:" + +#, fuzzy +#| msgid "Documents" +msgid "A document has been published" +msgstr "Dokumente" + +#, fuzzy +#| msgid "created" +msgid "rated" +msgstr "geskep" + +#, fuzzy +#| msgid "The following users:" +msgid "the following document" +msgstr "Die volgende gebruikers:" + +msgid "You can visit the layer's detail page here" +msgstr "" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A document has been rated" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "The following users:" +msgid "The following document was updated" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A document has been updated" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "You have the following incomplete uploads" +msgid "You have received the following notice from" +msgstr "Jy het die volgende onvolledige oplaai" + +msgid "To change how you receive notifications, please go to " +msgstr "" + +#, python-format +msgid "[%(current_site)s] %(message)s" +msgstr "" + +#, fuzzy, python-format +#| msgid "Notices" +msgid "%(notice)s" +msgstr "Notices" + +#, fuzzy +#| msgid "The following layers will be imported" +msgid "The following layer was approved" +msgstr "Die volgende lae ingevoer word" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A layer has been approved" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "Documents related to this layer" +msgid "A comment has been posted for the layer" +msgstr "Dokumente wat verband hou met hierdie laag" + +#, fuzzy +#| msgid "A rating was given to a layer" +msgid "A comment has been posted for a layer" +msgstr "A-gradering is om 'n laag gegee" + +#, fuzzy +#| msgid "This map uses the following layers:" +msgid "uploaded the following layer" +msgstr "Hierdie kaart gebruik die volgende lae:" + +#, fuzzy +#| msgid "A layer was uploaded" +msgid "A new layer has been uploaded" +msgstr "'N laag is opgelaai" + +#, fuzzy +#| msgid "The following layers will be imported" +msgid "The following layer was deleted" +msgstr "Die volgende lae ingevoer word" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A layer has been deleted" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "The following layers will be imported" +msgid "The following layer was published" +msgstr "Die volgende lae ingevoer word" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A layer has been published" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "The following users:" +msgid "the following layer" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A layer has been rated" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "The following layers will be imported" +msgid "The following layer was updated" +msgstr "Die volgende lae ingevoer word" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A layer has been updated" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "The following layers will be imported" +msgid "The following map was approved" +msgstr "Die volgende lae ingevoer word" + +msgid "You can visit the map's detail page here" +msgstr "" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A map has been approved" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "Documents related to this map" +msgid "A comment has been posted for the map" +msgstr "Dokumente wat verband hou met die kaart" + +#, fuzzy +#| msgid "Documents related to this map" +msgid "A comment has been posted for a map" +msgstr "Dokumente wat verband hou met die kaart" + +#, fuzzy +#| msgid "The following users:" +msgid "created the following map" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A map has been created" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "The following layers will be imported" +msgid "The following map was deleted" +msgstr "Die volgende lae ingevoer word" + +#, fuzzy +#| msgid "A map was commented on" +msgid "A map has been deleted" +msgstr "'N kaart is kommentaar op" + +#, fuzzy +#| msgid "The following users:" +msgid "The following map was published" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "A layer was commented on" +msgid "A map has been published" +msgstr "'N laag is kommentaar op" + +#, fuzzy +#| msgid "The following users:" +msgid "the following map" +msgstr "Die volgende gebruikers:" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A map has been rated" +msgstr "Diens is geskep:" + +#, fuzzy +#| msgid "The following layers will be imported" +msgid "The following map was updated" +msgstr "Die volgende lae ingevoer word" + +#, fuzzy +#| msgid "A map was commented on" +msgid "A map has been updated" +msgstr "'N kaart is kommentaar op" + +msgid "Notification Settings" +msgstr "Beriginstellings" + +#, python-format +msgid "" +"\n" +"

          \n" +" Note:\n" +" You do not have a verified email address to which notices can be " +"sent. Add one now.\n" +"

          \n" +" " +msgstr "" + +msgid "Notification Type" +msgstr "Kennisgewing Tipe" + +#, fuzzy +#| msgid "party that owns the resource" +msgid "requested you to download this resource" +msgstr "party wat die eienaar van die hulpbron" + +msgid "" +"Please go to resource page and assign the download permissions if you wish" +msgstr "" + +#, fuzzy +#| msgid "Service has been created:" +msgid "A resource's download has been requested" +msgstr "Diens is geskep:" + +msgid "Privacy & Cookies Policy" +msgstr "" + +msgid "Privacy & Cookies Policy." +msgstr "" + +msgid "Following" +msgstr "Volgende" + +msgid "This user is not following anyone" +msgstr "Hierdie gebruiker is nie as gevolg van iemand" + +msgid "Followers" +msgstr "Volgelinge" + +msgid "No one is following this user" +msgstr "Niemand is na aanleiding van hierdie gebruiker" + +msgid "Manage Connections" +msgstr "Bestuur Connections" + +msgid "Blocking" +msgstr "Blokkeer" + +msgid "Unfollow" +msgstr "Unfollow" + +msgid "Follow" +msgstr "Volg" + +msgid "Unblock" +msgstr "Blokkeer" + +msgid "Block" +msgstr "Blok" + +msgid "No longer" +msgstr "Nie meer" + +#, python-format +msgid "You're not %(status_slug)s anyone!" +msgstr "Jy is nie %(status_slug)s iemand!" + +msgid "Success" +msgstr "Sukses" + +msgid "You are " +msgstr "Jy is" + +msgid "no longer" +msgstr "nie meer" + +#, python-format +msgid "%(to_user_username)s's profile" +msgstr "%(to_user_username)s se profiel" + +msgid "Responsibles" +msgstr "" + +msgid "Date begins after:" +msgstr "Datum begin na:" + +msgid "yyyy-mm-dd" +msgstr "yyyy-mm-dd" + +msgid "Date ends before:" +msgstr "Datum eindig voor:" + +#, fuzzy +#| msgid "Search" +msgid "Search by region" +msgstr "Soek" + +#, fuzzy +#| msgid "Sections" +msgid "Selection" +msgstr "Artikels" + +msgid "No list items selected. Use the selection fields to add." +msgstr "" + +msgid "Filters" +msgstr "" + +#, fuzzy +#| msgid "Layers" +msgid "Datasets found" +msgstr "Lae" + +msgid "Maps found" +msgstr "" + +#, fuzzy +#| msgid "Documents" +msgid "Documents found" +msgstr "Dokumente" + +msgid "found" +msgstr "" + +msgid "search" +msgstr "" + +#, fuzzy +#| msgid "Search" +msgid "Search by name" +msgstr "Soek" + +msgid "Total" +msgstr "" + +msgid "Most recent" +msgstr "Mees onlangse" + +msgid "Less recent" +msgstr "Minder onlangse" + +msgid "A - Z" +msgstr "A - Z" + +msgid "Z - A" +msgstr "Z - A" + +msgid "Most popular" +msgstr "Gewildste" + +msgid "Text" +msgstr "Teks" + +#, fuzzy +#| msgid "feature type" +msgid "Share This Dataset" +msgstr "funksie tipe" + +msgid "Share This" +msgstr "" + +msgid "Social Network Login Failure" +msgstr "" + +msgid "" +"An error occurred while attempting to login via your social network account." +msgstr "" + +msgid "Code" +msgstr "" + +#, fuzzy +#| msgid "Sections" +msgid "Exception" +msgstr "Artikels" + +#, fuzzy +#| msgid "Manage Connections" +msgid "Account Connections" +msgstr "Bestuur Connections" + +msgid "" +"You can sign in to your account using any of the following already connected " +"third party accounts:" +msgstr "" + +msgid "" +"You currently have no social network accounts connected to this account." +msgstr "" + +msgid "Add a 3rd Party Account" +msgstr "" + +msgid "Login Cancelled" +msgstr "" + +#, python-format +msgid "" +"You decided to cancel logging in to our site using one of your existing " +"accounts. If this was a mistake, please proceed to sign in." +msgstr "" + +#, fuzzy +#| msgid "Sign up" +msgid "Signup" +msgstr "Sluit aan" + +#, fuzzy +#| msgid "Sign up" +msgid "Sign Up" +msgstr "Sluit aan" + +#, python-format +msgid "" +"You are about to use your %(provider_name)s account to login to\n" +"%(site_name)s. As a final step, please complete the following form:" +msgstr "" + +#, fuzzy +#| msgid "Sign in" +msgid "Sign in with" +msgstr "Teken in" + +msgid "Connect with" +msgstr "" + +#, fuzzy +#| msgid "Sign up" +msgid "Sign up with" +msgstr "Sluit aan" + +msgid "Message Inbox" +msgstr "Boodskap Inbox" + +msgid "Messages" +msgstr "" + +msgid "Mark selected as read" +msgstr "Merk gekies as gelees" + +msgid "Mark selected as unread" +msgstr "Merk as ongelees gekies" + +msgid "Create Message" +msgstr "Skep boodskap" + +#, fuzzy +#| msgid "Choose users..." +msgid "To users" +msgstr "Kies gebruikers ..." + +#, fuzzy +#| msgid "group" +msgid "To groups" +msgstr "groep" + +msgid "Subject" +msgstr "Onderwerp" + +#, fuzzy +#| msgid "Contact:" +msgid "Content" +msgstr "Kontak:" + +msgid "Send message" +msgstr "Stuur boodskap" + +msgid "Back to Inbox" +msgstr "Terug na Inbox" + +msgid "Sent to" +msgstr "stuur na" + +msgid "by" +msgstr "" + +#, fuzzy +#| msgid "Send message" +msgid "You have no messages" +msgstr "Stuur boodskap" + +msgid "Send Reply" +msgstr "Stuur Antwoord" + +#, fuzzy +#| msgid "GeoNode Help" +msgid "GeoNode Themes Library" +msgstr "GeoNode Hulp" + +msgid "Fill in this section with markdown" +msgstr "" + +msgid "Check this if the jumbotron background image already contains text" +msgstr "" + +msgid "Disabling this slide will hide it from the slide show" +msgstr "" + +msgid "Choose between using jumbotron background and slide show" +msgstr "" + +#, fuzzy +#| msgid "All contents" +msgid "Empty content" +msgstr "Alle inhoud" + +#, fuzzy +#| msgid "Could not find any layers in a compatible projection." +msgid "Could not access to uploaded data." +msgstr "Kon nie enige lae in 'n aanpasbare projeksie vind." + +msgid "" +"One or more XML files was provided, but no matching files were found for " +"them." +msgstr "" + +msgid "" +"One or more SLD files was provided, but no matching files were found for " +"them." +msgstr "" + +#, fuzzy +#| msgid "Map Created" +msgid "Layer already exists" +msgstr "Kaart geskep" + +msgid "Name already in use and overwrite is False" +msgstr "" + +msgid "Error configuring time:" +msgstr "" + +msgid "The GeoServer Import Session is no more available" +msgstr "" + +#, python-brace-format +msgid "Expected to find layer named '{name}' in geoserver" +msgstr "" + +msgid "Exception occurred while parsing the provided Metadata file." +msgstr "" + +msgid "" +"The UUID identifier from the XML Metadata is already in use in this system." +msgstr "" + +msgid "Error configuring Layer" +msgstr "" + +msgid "Invalid zip file detected" +msgstr "" + +#, fuzzy +#| msgid "Could not find any layers in a compatible projection." +msgid "Could not find any valid spatial file inside the uploaded zip" +msgstr "Kon nie enige lae in 'n aanpasbare projeksie vind." + +msgid "Invalid kmz file detected" +msgstr "" + +#, fuzzy +#| msgid "Could not find any layers in a compatible projection." +msgid "Could not find any kml files inside the uploaded kmz" +msgstr "Kon nie enige lae in 'n aanpasbare projeksie vind." + +msgid "Only one shapefile per zip is allowed" +msgstr "" + +msgid "kml files with more than one GroundOverlay are not supported" +msgstr "" + +msgid "Ground overlay image declared in kml file cannot be found" +msgstr "" + +msgid "Only one kml file per ZIP is allowed" +msgstr "" + +msgid "Only one kml file per kmz is allowed" +msgstr "" + +msgid "" +"You are trying to upload multiple GeoTIFFs without a valid 'indexer." +"properties' file." +msgstr "" + +msgid "Only one raster file per ZIP is allowed" +msgstr "" + +msgid "No multiple rasters allowed" +msgstr "" + +msgid "" +"To support the time step, you must enable the OGC_SERVER DATASTORE option" +msgstr "" + +#, python-brace-format +msgid "Unsupported file type: {e.message}" +msgstr "" + +#, python-brace-format +msgid "File does not exist: {absolute_base_file}" +msgstr "" + +#, python-brace-format +msgid "unknown item state: {task.state}" +msgstr "" + +msgid "error during import: {progress.get('message')}" +msgstr "" + +#, fuzzy +#| msgid "Could not find any layers in a compatible projection." +msgid "Could not find any valid Time Regex for the Mosaic files." +msgstr "Kon nie enige lae in 'n aanpasbare projeksie vind." + +msgid "Unsupported DataBase for Mosaics!" +msgstr "" + +msgid "Upload Session invalid or no more accessible!" +msgstr "" + +msgid "Invalid permission level." +msgstr "Ongeldig toestemming vlak." + +msgid "Permission Denied" +msgstr "Toestemming geweier" + +#, fuzzy +#~| msgid "Layer Name" +#~ msgid "Layer name" +#~ msgstr "Laag Naam" + +#, fuzzy +#~| msgid "Layer Title" +#~ msgid "Layer title" +#~ msgstr "Laag titel" + +#, fuzzy +#~| msgid "Create a Map" +#~ msgid "Create Layer" +#~ msgstr "Skep 'n kaart" + +#~ msgid "Download Layer" +#~ msgstr "Aflaai Layer" + +#, fuzzy +#~| msgid "Edit Layer" +#~ msgid "View Layer" +#~ msgstr "Wysig Layer" + +#, fuzzy +#~| msgid "Permissions" +#~ msgid "Updating Permissions..." +#~ msgstr "Permissions" + +#~ msgid "Upload Layers" +#~ msgstr "Oplaai Lae" + +#~ msgid "Upload Layer" +#~ msgstr "Oplaai Layer" + +#, fuzzy +#~| msgid "Files to be uploaded" +#~ msgid "Failed to upload the layer" +#~ msgstr "Lêers opgelaai word" + +#~ msgid "Register a new Service" +#~ msgstr "Registreer 'n nuwe diens" + +#, fuzzy +#~| msgid "Set permissions" +#~ msgid "set permissions" +#~ msgstr "Stel regte" + +#, fuzzy +#~| msgid "Documents" +#~ msgid "documents" +#~ msgstr "Dokumente" + +#, fuzzy +#~| msgid "dataset" +#~ msgid "data" +#~ msgstr "dataset" + +#, fuzzy +#~| msgid "GeoExplorer" +#~ msgid "apps explore" +#~ msgstr "GeoExplorer" + +#, fuzzy +#~| msgid "Categories" +#~ msgid "groups explore" +#~ msgstr "Kategorieë" + +#, fuzzy +#~| msgid "A Map was created" +#~ msgid "groups create" +#~ msgstr "'N Plan is geskep" + +#, fuzzy +#~| msgid "group" +#~ msgid "groups" +#~ msgstr "groep" + +#, fuzzy +#~| msgid "updated" +#~ msgid "groups update" +#~ msgstr "updated" + +#, fuzzy +#~| msgid "A layer was uploaded" +#~ msgid "layers append" +#~ msgstr "'N laag is opgelaai" + +#, fuzzy +#~| msgid "Layers" +#~ msgid "layers" +#~ msgstr "Lae" + +#, fuzzy +#~| msgid "A layer was uploaded" +#~ msgid "layers explore" +#~ msgstr "'N laag is opgelaai" + +#, fuzzy +#~| msgid "A layer was uploaded" +#~ msgid "layers upload" +#~ msgstr "'N laag is opgelaai" + +#~ msgid "Incomplete Uploads" +#~ msgstr "Onvolledige Uploads" + +#~ msgid "You have the following incomplete uploads" +#~ msgstr "Jy het die volgende onvolledige oplaai" + +#~ msgid "Resume" +#~ msgstr "CV" + +#, fuzzy +#~| msgid "GeoExplorer" +#~ msgid "maps explore" +#~ msgstr "GeoExplorer" + +#, fuzzy +#~| msgid "GeoExplorer" +#~ msgid "people explore" +#~ msgstr "GeoExplorer" + +#, fuzzy +#~| msgid "Invite people" +#~ msgid "edit people" +#~ msgstr "Nooi mense" + +#~ msgid "last updated on" +#~ msgstr "laas op" + +#~ msgid "Delete, and don't ask me again." +#~ msgstr "Verwyder, en my nie weer vra nie." + +#, fuzzy +#~| msgid "Available styles" +#~ msgid "Available layers" +#~ msgstr "Beskikbaar style" + +#, fuzzy +#~| msgid "Create a Map" +#~ msgid "Save Map" +#~ msgstr "Skep 'n kaart" + +#, fuzzy +#~| msgid "User Activities" +#~ msgid "Utilities" +#~ msgstr "Gebruikers Aktiwiteite" + +#, fuzzy +#~| msgid "Files to be uploaded" +#~ msgid "Unable to delete layer" +#~ msgstr "Lêers opgelaai word" + +#, fuzzy +#~| msgid "Thumbnail" +#~ msgid "Thumbnail saved" +#~ msgstr "Thumbnail" diff --git a/geonode/locale/af/LC_MESSAGES/djangojs.mo b/geonode/locale/af/LC_MESSAGES/djangojs.mo new file mode 100644 index 0000000..951fa77 Binary files /dev/null and b/geonode/locale/af/LC_MESSAGES/djangojs.mo differ diff --git a/geonode/locale/af/LC_MESSAGES/djangojs.po b/geonode/locale/af/LC_MESSAGES/djangojs.po new file mode 100644 index 0000000..708ed59 --- /dev/null +++ b/geonode/locale/af/LC_MESSAGES/djangojs.po @@ -0,0 +1,1039 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# ingenieroariel , 2014 +# ingenieroariel , 2014 +msgid "" +msgstr "" +"Project-Id-Version: GeoNode\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-07-08 09:03+0300\n" +"PO-Revision-Date: 2016-08-23 11:58+0000\n" +"Last-Translator: Julien Collaer \n" +"Language-Team: Afrikaans (http://www.transifex.com/geonode/geonode/language/" +"af/)\n" +"Language: af\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +msgid "STORE_UPDATED" +msgstr "" + +#, javascript-format +msgid "Missing a %s file, which is required" +msgstr "Ontbreek 'n %s lêer, wat nodig is" + +msgid "ESRI Shapefile" +msgstr "ESRI Shape File" + +msgid "ASCII Text File" +msgstr "" + +msgid "GeoTIFF" +msgstr "" + +msgid "Comma Separated Value" +msgstr "" + +msgid "Google Earth KML" +msgstr "Google Earth KML" + +#, fuzzy +#| msgid "Google Earth KML" +msgid "Google Earth KML with a GroundOverlay" +msgstr "Google Earth KML" + +#, fuzzy +#| msgid "Google Earth KML" +msgid "Google Earth KMZ" +msgstr "Google Earth KML" + +msgid "GeoJSON" +msgstr "GeoJSON" + +msgid "Zip Archive" +msgstr "" + +msgid "ERDAS Imagine" +msgstr "" + +msgid "National Imagery Transmission Format" +msgstr "" + +msgid "Controlled Image Base-1 Meter" +msgstr "" + +msgid "Controlled Image Base-5 Meter" +msgstr "" + +msgid "Controlled Image Base-10 Meter" +msgstr "" + +msgid "CADRG-Global Navigation Chart" +msgstr "" + +msgid "CADRG-Jet Navigation Chart" +msgstr "" + +msgid "CADRG-Operational Navigation Chart" +msgstr "" + +msgid "CADRG-Tactical Pilotage Chart" +msgstr "" + +msgid "CADRG-Joint Operations Graphic" +msgstr "" + +msgid "CADRG-Topographic Line Map 100K" +msgstr "" + +msgid "CADRG-Topographic Line Map 50K" +msgstr "" + +msgid "JPEG2000" +msgstr "" + +msgid "MrSID-Multi-resolution Seamless Image Database" +msgstr "" + +#, fuzzy +#| msgid "Metadata Links:" +msgid "XML Metadata File" +msgstr "Metadata Links:" + +msgid "Style Layer Descriptor" +msgstr "" + +msgid "JPEG" +msgstr "" + +msgid "PNG" +msgstr "" + +msgid "GIF" +msgstr "" + +#, fuzzy +#| msgid "Unexpected Error" +msgid "Unexpected error!" +msgstr "Onverwagse fout" + +msgid "" +" - 400 Bad Request. Server cannot or will not process the request due to " +"something that is perceived to be a client error (e.g., malformed request " +"syntax, invalid request message framing, or deceptive request routing)." +msgstr "" + +msgid "" +" - 401 Unauthorized. Request was not sent with the proper authentication " +"credentials." +msgstr "" + +msgid "" +" - 403 Forbidden. This is generally related to permission rules on your " +"server. Contact the system administrator for more information regarding this " +"error message." +msgstr "" + +msgid "" +" - 404 Not Found. Origin server was unable or unwilling to find the resource " +"requested." +msgstr "" + +msgid "" +" - 405 Method Not Allowed. Origin server is aware of the requested resource, " +"but the request method used is not supported." +msgstr "" + +msgid "" +" - 406 Not Acceptable. Resource is not available at the origin that adheres " +"to negotiation headers that were set prior (e.g. via 'Accept-Charset' and " +"'Accept-Language' headers)." +msgstr "" + +msgid "" +" - 407 Authentication Required. The client did not send the required " +"authentication with the request." +msgstr "" + +msgid "" +" - 408 Request Timeout. The origin server did not receive the complete " +"request in what it considers a reasonable time." +msgstr "" + +msgid "" +" - 409 Conflict. The request did not complete because of a conflict with the " +"current state of the resource. Typically happens on a PUT request where " +"multiple clients are attempting to edit the same resource." +msgstr "" + +msgid "" +" - 410 Gone. The resource requested is permanently missing at the origin." +msgstr "" + +msgid "" +" - 411 Length Required. Client did not define the 'Content-Length' of the " +"request body in the headers and this is required to obtain the resource." +msgstr "" + +msgid "" +" - 412 Precondition Failed. Server denies the request because the resource " +"failed to meet the conditions specified by the client." +msgstr "" + +msgid "" +" - 413 Payload Too Large. Refusal from the server to process the request " +"because the payload sent from the client is larger than the server wished to " +"accept. Server has the optional to close the connection." +msgstr "" + +msgid "" +" - 414 URI Too Long. Refusal from the server that the URI was too long to be " +"processed. For example, if a client is attempting a GET request with an " +"unusually long URI after a POST, this could be seen as a security risk and a " +"414 gets generated." +msgstr "" + +msgid "" +" - 415 Unsupported Media Type. Refusal from the server to process the format " +"of the current payload. One way to identify and fix this issue would be to " +"look at the 'Content-Type' or 'Content-Encoding' headers sent in the " +"client’s request." +msgstr "" + +msgid "" +" - 417 Expectation Failed. Failure of server to meet the requirements " +"specified in the 'Expect' header of the client’s request." +msgstr "" + +msgid "" +" - 429 Too Many Requests. Client has sent too many requests in the specified " +"amount of time according to the server." +msgstr "" + +msgid "" +" - 499 Client Close Request. Nginx specific response code to indicate when " +"the connection has been closed by the client while the server is still " +"processing its request, making server unable to send a status code back." +msgstr "" + +msgid "" +" - 500 Internal Server Error. This error indicates that the server has " +"encountered an unexpected condition. This often occurs when an application " +"request cannot be fulfilled due to the application being configured " +"incorrectly on the server." +msgstr "" + +msgid "" +" - 501 Not Implemented. This error indicates that the HTTP method sent by " +"the client is not supported by the server. This is most often caused by the " +"server being out of date. It is a very rare error and generally requires " +"that the web server be updated." +msgstr "" + +msgid "" +" - 502 Bad Gateway. This error is usually due to improperly configured proxy " +"servers. The first step in resolving the issue is to clear the client's " +"cache." +msgstr "" + +msgid "" +" - 503 Service Unavailable. This error occurs when the server is unable to " +"handle requests due to a temporary overload or due to the server being " +"temporarily closed for maintenance. The error indicates that the server will " +"only temporarily be down." +msgstr "" + +msgid "" +" - 504 Gateway Timeout. GeoNode lost the connection with GeoServer or DB due " +"to a connection timeout. Consider using the management commands to import " +"data!" +msgstr "" + +msgid "" +" - 505 HTTP Version Not Supported. This error occurs when the server refuses " +"to support the HTTP protocol that has been specified by the client computer. " +"This can be caused by the protocol not being specified properly by the " +"client computer; for example, if an invalid version number has been " +"specified." +msgstr "" + +msgid "" +" - 506 Variant Also Negotiates. This error indicates that the server is not " +"properly configured. Contact the system administrator to resolve this issue." +msgstr "" + +msgid "" +" - 507 Insufficient Storage. This error indicates that the server is out of " +"free memory. This is most likely to occur when an application that is being " +"requested cannot allocate the necessary system resources to run. To resolve " +"the issue, the server's hard disk may need to be cleaned of any unnecessary " +"documents to free up more hard disk space, its memory may need to be " +"expanded, or it may simply need to be restarted. Contact the system " +"administrator for more information regarding this error message." +msgstr "" + +msgid "" +" - 509 Bandwidth Limit Exceeded. This error occurs when the bandwidth limit " +"imposed by the system administrator has been reached. The only fix for this " +"issue is to wait until the limit is reset in the following cycle. Consult " +"the system administrator for information about acquiring more bandwidth." +msgstr "" + +msgid "" +" - 510 Not Extended. This error occurs when an extension attached to the " +"HTTP request is not supported by the web server. To resolve the issue, you " +"may need to update the server." +msgstr "" + +#, fuzzy +#| msgid "Unknown Map" +msgid "Unknwon" +msgstr "Onbekend Map" + +msgid "" +" Error Code. Contact the system administrator for more information regarding " +"this error message." +msgstr "" + +msgid "Additional info: " +msgstr "" + +msgid "Finalizing" +msgstr "" + +msgid "Unexpected Error" +msgstr "Onverwagse fout" + +msgid "Your " +msgstr "" + +#, fuzzy +#| msgid "Layer Info" +msgid " Info" +msgstr "Laag Info" + +msgid "Edit Metadata" +msgstr "Wysig Metadata" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "Upload Metadata" +msgstr "Wysig Metadata" + +msgid "Upload SLD" +msgstr "" + +msgid "Manage Styles" +msgstr "Bestuur Styles" + +msgid "Please wait until GeoNode finished configuring it!" +msgstr "" + +msgid "The column %1 was renamed to %2
          " +msgstr "" + +msgid "Performing Final GeoServer Config Step" +msgstr "Presterende Finale Geoserver Config Stap" + +msgid "You need to specify more information in order to complete your upload" +msgstr "Jy moet meer inligting te gee om jou oplaai om te voltooi" + +msgid "Files are ready to be ingested!" +msgstr "" + +msgid "A temporal dimension may be added to this Layer." +msgstr "" + +msgid "Performing GeoServer Config Step" +msgstr "Presterende Geoserver Config Stap" + +msgid "" +"
          Please check your network connection. In case of Layer Upload make sure " +"GeoServer is running and accepting connections." +msgstr "" + +msgid "
          " +msgstr "" + +msgid "
          Bad request or URL not found." +msgstr "" + +#, fuzzy +#| msgid "Unknown Map" +msgid "
          Unknown." +msgstr "Onbekend Map" + +msgid "Layer files uploaded, configuring in GeoServer" +msgstr "Laag lêers opgelaai, instel in Geoserver" + +msgid "Remove" +msgstr "Verwyder" + +#, javascript-format +msgid "The file %s is an unsupported file type, please select another file." +msgstr "" +"Die lêer %s is 'n nie-ondersteunde lêer tipe, kies asseblief 'n ander lêer." + +msgid "" +"You are trying to upload an incomplete set of files or not all mandatory " +"options have been validated.\n" +"\n" +"Please check for errors in the form!" +msgstr "" + +#, javascript-format +msgid "Available %s" +msgstr "Available %s" + +#, javascript-format +msgid "" +"This is the list of available %s. You may choose some by selecting them in " +"the box below and then clicking the \"Choose\" arrow between the two boxes." +msgstr "" +"This is the list of available %s. You may choose some by selecting them in " +"the box below and then clicking the \"Choose\" arrow between the two boxes." + +#, javascript-format +msgid "Type into this box to filter down the list of available %s." +msgstr "Type into this box to filter down the list of available %s." + +msgid "Filter" +msgstr "Filter" + +msgid "Choose all" +msgstr "Choose all" + +#, javascript-format +msgid "Click to choose all %s at once." +msgstr "Click to choose all %s at once." + +msgid "Choose" +msgstr "Choose" + +#, javascript-format +msgid "Chosen %s" +msgstr "Chosen %s" + +#, javascript-format +msgid "" +"This is the list of chosen %s. You may remove some by selecting them in the " +"box below and then clicking the \"Remove\" arrow between the two boxes." +msgstr "" +"This is the list of chosen %s. You may remove some by selecting them in the " +"box below and then clicking the \"Remove\" arrow between the two boxes." + +msgid "Remove all" +msgstr "Remove all" + +#, javascript-format +msgid "Click to remove all chosen %s at once." +msgstr "Click to remove all chosen %s at once." + +msgid "%(sel)s of %(cnt)s selected" +msgid_plural "%(sel)s of %(cnt)s selected" +msgstr[0] "94d7ef36d54e81afde607d8fd121772b_pl_0" +msgstr[1] "94d7ef36d54e81afde607d8fd121772b_pl_1" + +msgid "" +"You have unsaved changes on individual editable fields. If you run an " +"action, your unsaved changes will be lost." +msgstr "" +"You have unsaved changes on individual editable fields. If you run an " +"action, your unsaved changes will be lost." + +msgid "" +"You have selected an action, but you haven't saved your changes to " +"individual fields yet. Please click OK to save. You'll need to re-run the " +"action." +msgstr "" +"You have selected an action, but you haven't saved your changes to " +"individual fields yet. Please click OK to save. You'll need to re-run the " +"action." + +msgid "" +"You have selected an action, and you haven't made any changes on individual " +"fields. You're probably looking for the Go button rather than the Save " +"button." +msgstr "" +"You have selected an action, and you haven't made any changes on individual " +"fields. You're probably looking for the Go button rather than the Save " +"button." + +msgid "'" +msgstr "" + +msgid "-1" +msgstr "" + +msgid "trailingComments" +msgstr "" + +msgid "leadingComments" +msgstr "" + +#, fuzzy +#| msgid "Enter comments here." +msgid "innerComments" +msgstr "Gee kommentaar hier." + +msgid "Multiselect" +msgstr "" + +#, fuzzy +#| msgid "Select" +msgid "SelectList" +msgstr "Kies" + +msgid "show:" +msgstr "" + +msgid "DraftEditor/root" +msgstr "" + +msgid "DraftEditor/editorContainer" +msgstr "" + +msgid "-" +msgstr "" + +msgid "uirevision" +msgstr "" + +msgid "bgcolor" +msgstr "" + +msgid "bordercolor" +msgstr "" + +msgid "borderwidth" +msgstr "" + +msgid "orientation" +msgstr "" + +msgid "traceorder" +msgstr "" + +msgid "tracegroupgap" +msgstr "" + +msgid "itemsizing" +msgstr "" + +msgid "itemclick" +msgstr "" + +msgid "itemdoubleclick" +msgstr "" + +msgid "x" +msgstr "" + +msgid "xanchor" +msgstr "" + +msgid "y" +msgstr "" + +msgid "yanchor" +msgstr "" + +msgid "valign" +msgstr "" + +msgid "title.text" +msgstr "" + +msgid "title.side" +msgstr "" + +msgid "visible" +msgstr "" + +msgid "thickness" +msgstr "" + +msgid "autorange" +msgstr "" + +msgid "range" +msgstr "" + +msgid "bingroup" +msgstr "" + +msgid "Print configuration broken (" +msgstr "" + +msgid "Print configuration not available (" +msgstr "" + +msgid "go" +msgstr "" + +msgid "goBack" +msgstr "" + +msgid "goForward" +msgstr "" + +msgid "Abort" +msgstr "" + +msgid "As Sibling" +msgstr "" + +msgid "As child" +msgstr "" + +#, fuzzy +#~| msgid "Now" +#~ msgid "No" +#~ msgstr "Now" + +#~ msgid "Your layer was successfully uploaded" +#~ msgstr "Jou laag is suksesvol opgelaai" + +#~ msgid "" +#~ "You need to specify more information in order to complete your upload." +#~ msgstr "Jy moet meer inligting te gee om jou oplaai om te voltooi." + +#~ msgid "You can continue configuring your layer." +#~ msgstr "Jy kan voortgaan om die instel van jou laag." + +#~ msgid "Cancel" +#~ msgstr "Kanselleer" + +#~ msgid "Save as Copy" +#~ msgstr "Red as kopieer" + +#~ msgid "About this Map" +#~ msgstr "Oor die kaart" + +#~ msgid "Abstract" +#~ msgstr "Abstrakte" + +#~ msgid "Save" +#~ msgstr "Save" + +#~ msgid "Title" +#~ msgstr "Titel" + +#~ msgid "Connection Error" +#~ msgstr "Verband Fout" + +#~ msgid "The server returned an error" +#~ msgstr "Die bediener het 'n fout" + +#~ msgid "Details..." +#~ msgstr "Besonderhede ..." + +#~ msgid "Add Layers" +#~ msgstr "Voeg Lae" + +#~ msgid "Area" +#~ msgstr "Area" + +#~ msgid "Background" +#~ msgstr "Agtergrond" + +#~ msgid "Done" +#~ msgstr "Gedaan" + +#~ msgid "Available Layers" +#~ msgstr "Beskikbare lae" + +#~ msgid "Height" +#~ msgstr "Hoogte" + +#~ msgid "Get Feature Info" +#~ msgstr "Kry Kenmerk Info" + +#~ msgid "Large" +#~ msgstr "Groot" + +#~ msgid "or add a new server." +#~ msgstr "of voeg 'n nuwe bediener." + +#~ msgid "Map Layers" +#~ msgstr "Kaartlae" + +#~ msgid "View available data from:" +#~ msgstr "Kyk beskikbare data van:" + +#~ msgid "Data" +#~ msgstr "Data" + +#~ msgid "Layers" +#~ msgstr "Lae" + +#~ msgid "Legend" +#~ msgstr "Legende" + +#~ msgid "Length" +#~ msgstr "Lengte" + +#~ msgid "Loading Map..." +#~ msgstr "Plek ..." + +#~ msgid "Map Size" +#~ msgstr "Kaart grootte" + +#~ msgid "Measure" +#~ msgstr "Maatreël" + +#~ msgid "Contact" +#~ msgstr "Kontak" + +#~ msgid "Permalink" +#~ msgstr "Permalink" + +#~ msgid "Mini" +#~ msgstr "Mini" + +#~ msgid "Pan Map" +#~ msgstr "Pan-oorsig" + +#~ msgid "Zoom to Next Extent" +#~ msgstr "Zoom Volgende Mate" + +#~ msgid "Zoom to Previous Extent" +#~ msgstr "Zoom Vorige Mate" + +#~ msgid "This map has not yet been saved." +#~ msgstr "Hierdie kaart is nog nie gestoor nie." + +#~ msgid "Premium" +#~ msgstr "Premium" + +#~ msgid "Print Map" +#~ msgstr "Druk Map" + +#~ msgid "Print Preview" +#~ msgstr "Print Preview" + +#~ msgid "Properties" +#~ msgstr "Eiendomme" + +#~ msgid "Publish Map" +#~ msgstr "Publiseer Map" + +#~ msgid "Remove Layer" +#~ msgstr "Verwyder Layer" + +#~ msgid "Sorry, your map could not be saved." +#~ msgstr "Jammer, jou kaart kan nie gered word nie." + +#~ msgid "Error While Saving" +#~ msgstr "Fout terwyl spaar" + +#~ msgid "Save Map" +#~ msgstr "Save Map" + +#~ msgid "You must be logged in to save this map." +#~ msgstr "Jy moet aangemeld wees om die kaart te red." + +#~ msgid "Small" +#~ msgstr "Klein" + +#~ msgid "" +#~ "Error contacting server.\n" +#~ " Please check the url and try again." +#~ msgstr "Fout kontak bediener. Gaan asseblief die URL na en probeer weer." + +#~ msgid "Layer Properties" +#~ msgstr "Laag Properties" + +#~ msgid "Change layer format and style" +#~ msgstr "Verandering laag formaat en styl" + +#~ msgid "Edit Styles" +#~ msgstr "Wysig Styles" + +#~ msgid "Edit layer styles" +#~ msgstr "Wysig laag style" + +#~ msgid "Switch to Google Earth 3D Viewer" +#~ msgstr "Skakel oor na Google Earth 3D Viewer" + +#~ msgid "" +#~ "The map that you are trying to load does not exist. Creating a new map " +#~ "instead." +#~ msgstr "" +#~ "Die kaart wat jy probeer om te laai nie bestaan ​​nie. Skep van 'n nuwe " +#~ "kaart plaas." + +#~ msgid "Width" +#~ msgstr "Wydte" + +#~ msgid "Zoom In" +#~ msgstr "Klik op In" + +#~ msgid "Zoom Out" +#~ msgstr "Verklein" + +#~ msgid "Zoom level" +#~ msgstr "Klik op die vlak" + +#~ msgid "Zoom Level" +#~ msgstr "Klik op die vlak" + +#~ msgid "Zoom to Layer Extent" +#~ msgstr "Zoom Mate Pak" + +#~ msgid "Zoom to Visible Extent" +#~ msgstr "Zoom sigbaar Mate" + +#~ msgid "Zoom Level {zoom}" +#~ msgstr "Klik op die vlak {zoom}" + +#~ msgid "Create Map" +#~ msgstr "Skep Map" + +#~ msgid "Export Map" +#~ msgstr "Uitvoer Map" + +#~ msgid "Map" +#~ msgstr "Kaart" + +#~ msgid "View this Map" +#~ msgstr "Kyk die kaart" + +#~ msgid "Open Map" +#~ msgstr "Open Plan" + +#~ msgid "Name" +#~ msgstr "Naam" + +#~ msgid "Queryable" +#~ msgstr "Queryable" + +#~ msgid "No background" +#~ msgstr "Geen agtergrond" + +#~ msgid "" +#~ "Your map is ready to be published to the web! Simply copy the following " +#~ "HTML to embed the map in your website:" +#~ msgstr "" +#~ "Jou kaart gereed is om gepubliseer word op die web! Kopieer die volgende " +#~ "HTML die kaart in jou webwerf te sluit:" + +#~ msgid "Abstract:" +#~ msgstr "Opsomming:" + +#~ msgid "No attribution information is provided for this layer." +#~ msgstr "Geen toeskrywing inligting word verskaf vir hierdie laag." + +#~ msgid "Provided by:" +#~ msgstr "Verskaf deur:" + +#~ msgid "Download:" +#~ msgstr "Download:" + +#~ msgid "No keywords are listed for this layer." +#~ msgstr "Geen dokumente is gelys vir hierdie laag." + +#~ msgid "Keywords:" +#~ msgstr "Sleutelwoorde:" + +#~ msgid "No metadata URLs are defined for this layer." +#~ msgstr "Geen metadata URLs is vir hierdie laag gedefinieer." + +#~ msgid "Paper size:" +#~ msgstr "Papier grootte:" + +#~ msgid "Resolution:" +#~ msgstr "Resolusie:" + +#~ msgid "Print" +#~ msgstr "Print" + +#~ msgid "Enter map title here." +#~ msgstr "Tik kaart titel hier." + +#~ msgid "Include legend?" +#~ msgstr "Sluit legende?" + +#~ msgid "Creating PDF..." +#~ msgstr "Skep PDF ..." + +#~ msgid "Last Modified" +#~ msgstr "Last Modified" + +#~ msgid "Prev" +#~ msgstr "Terug" + +#~ msgid "Next" +#~ msgstr "Volgende" + +#~ msgid "of" +#~ msgstr "van" + +#~ msgid "Your search did not match any items." +#~ msgstr "Jou soektog het geen ooreenstemmende items." + +#~ msgid "Search Maps" +#~ msgstr "Soek Maps" + +#~ msgid "Search" +#~ msgstr "Soek" + +#~ msgid "Showing" +#~ msgstr "Resultate" + +#~ msgid "Loading" +#~ msgstr "Laai" + +#~ msgid "permalink" +#~ msgstr "permalink" + +#~ msgid "Select:" +#~ msgstr "Kies:" + +#~ msgid "All" +#~ msgstr "Alle" + +#~ msgid "None" +#~ msgstr "Geen" + +#~ msgid "Loading..." +#~ msgstr "Laai ..." + +#~ msgid "You do not have permission to view this data" +#~ msgstr "Jy het nie toestemming om hierdie inligting te besigtig het" + +#~ msgid "This data is stored on a remote server" +#~ msgstr "Hierdie data is op 'n afgeleë bediener gestoor" + +#~ msgid "No abstract is provided for this layer." +#~ msgstr "Geen abstrakte word vir hierdie laag." + +#~ msgid "No download URLs are definied for this layer." +#~ msgstr "Geen aflaai URLs word gedefinieer vir hierdie laag." + +#~ msgid "Selected Layers" +#~ msgstr "Gekies Lae" + +#~ msgid "No Layers Selected" +#~ msgstr "Geen lae Uitgesoekte" + +#~ msgid "Clear Selected" +#~ msgstr "Duidelik Uitgesoekte" + +#~ msgid "Clear All" +#~ msgstr "Duidelik al" + +#~ msgid "Operation Failed" +#~ msgstr "Misluk" + +#~ msgid "No layers are currently selected." +#~ msgstr "Geen lae tans gekies." + +#~ msgid "Downloading..." +#~ msgstr "Laai ..." + +#~ msgid "Please wait" +#~ msgstr "Wag asseblief" + +#~ msgid "" +#~ "January February March April May June July August September October " +#~ "November December" +#~ msgstr "" +#~ "January February March April May June July August September October " +#~ "November December" + +#~ msgid "S M T W T F S" +#~ msgstr "S M T W T F S" + +#~ msgid "Show" +#~ msgstr "Show" + +#~ msgid "Hide" +#~ msgstr "Hide" + +#~ msgid "Clock" +#~ msgstr "Clock" + +#~ msgid "Choose a time" +#~ msgstr "Choose a time" + +#~ msgid "Midnight" +#~ msgstr "Midnight" + +#~ msgid "6 a.m." +#~ msgstr "6 a.m." + +#~ msgid "Noon" +#~ msgstr "Noon" + +#~ msgid "Today" +#~ msgstr "Today" + +#~ msgid "Calendar" +#~ msgstr "Calendar" + +#~ msgid "Yesterday" +#~ msgstr "Yesterday" + +#~ msgid "Tomorrow" +#~ msgstr "Tomorrow" + +#~ msgid "Add user..." +#~ msgstr "Add user..." + +#~ msgid "Who can view and download this data?" +#~ msgstr "Who can view and download this data?" + +#~ msgid "Anyone" +#~ msgstr "Anyone" + +#~ msgid "Any registered user" +#~ msgstr "Any registered user" + +#~ msgid "Only users who can edit" +#~ msgstr "Only users who can edit" + +#~ msgid "Who can edit this data?" +#~ msgstr "Who can edit this data?" + +#~ msgid "Only the following users or groups:" +#~ msgstr "Only the following users or groups:" + +#~ msgid "Who can manage and edit this data?" +#~ msgstr "Who can manage and edit this data?" + +#~ msgid "GeoTiff File" +#~ msgstr "GeoTiff File" + +#~ msgid "Comma Separated File" +#~ msgstr "Comma Separated File" + +#~ msgid "Zip Archives" +#~ msgstr "Zip Archives" + +#~ msgid "You are uploading an incomplete set of files." +#~ msgstr "You are uploading an incomplete set of files." + +#~ msgid "No error message supplied" +#~ msgstr "No error message supplied" diff --git a/geonode/locale/al/LC_MESSAGES/django.mo b/geonode/locale/al/LC_MESSAGES/django.mo new file mode 100644 index 0000000..6c5906d Binary files /dev/null and b/geonode/locale/al/LC_MESSAGES/django.mo differ diff --git a/geonode/locale/al/LC_MESSAGES/django.po b/geonode/locale/al/LC_MESSAGES/django.po new file mode 100644 index 0000000..c7f7ba5 --- /dev/null +++ b/geonode/locale/al/LC_MESSAGES/django.po @@ -0,0 +1,5675 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-07-08 09:02+0300\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "Request to download a resource" +msgstr "" + +msgid "A request for downloading a resource was sent" +msgstr "" + +msgid "Request resource change" +msgstr "" + +msgid "Owner has requested permissions to modify a resource" +msgstr "" + +msgid "series" +msgstr "" + +msgid "computer program or routine" +msgstr "" + +msgid "feature type" +msgstr "" + +msgid "copy or imitation of an existing or hypothetical object" +msgstr "" + +msgid "collection hardware" +msgstr "" + +msgid "collection session" +msgstr "" + +msgid "non-geographic data" +msgstr "" + +msgid "property type" +msgstr "" + +msgid "field session" +msgstr "" + +msgid "dataset" +msgstr "" + +msgid "service interfaces" +msgstr "" + +msgid "attribute class" +msgstr "" + +msgid "characteristic of a feature" +msgstr "" + +msgid "tile or spatial subset of geographic data" +msgstr "" + +msgid "feature" +msgstr "" + +msgid "dimension group" +msgstr "" + +msgid "frequency of maintenance for the data is not known" +msgstr "" + +msgid "data is repeatedly and frequently updated" +msgstr "" + +msgid "there are no plans to update the data" +msgstr "" + +msgid "data is updated each day" +msgstr "" + +msgid "data is updated every year" +msgstr "" + +msgid "data is updated as deemed necessary" +msgstr "" + +msgid "data is updated each month" +msgstr "" + +msgid "data is updated every two weeks" +msgstr "" + +msgid "data is updated in intervals that are uneven in duration" +msgstr "" + +msgid "data is updated on a weekly basis" +msgstr "" + +msgid "data is updated twice each year" +msgstr "" + +msgid "data is updated every three months" +msgstr "" + +msgid "No information provided" +msgstr "" + +msgid "Category is required." +msgstr "" + +msgid "Keywords from Thesaurus" +msgstr "" + +msgid "List of keywords from Thesaurus" +msgstr "" + +msgid "Abstract" +msgstr "" + +msgid "Purpose" +msgstr "" + +msgid "Other constraints" +msgstr "" + +msgid "Supplemental information" +msgstr "" + +msgid "Data quality statement" +msgstr "" + +msgid "Owner" +msgstr "" + +msgid "Date" +msgstr "" + +msgid "temporal extent start" +msgstr "" + +msgid "temporal extent end" +msgstr "" + +msgid "Person outside GeoNode (fill form)" +msgstr "" + +msgid "Point of Contact" +msgstr "" + +msgid "Metadata Author" +msgstr "" + +msgid "Free-text Keywords" +msgstr "" + +msgid "" +"A space or comma-separated list of keywords. Use the widget to select from " +"Hierarchical tree." +msgstr "" + +msgid "Regions" +msgstr "" + +msgid "Group" +msgstr "" + +msgid "Category" +msgstr "" + +msgid "License" +msgstr "" + +msgid "Language" +msgstr "" + +msgid "User" +msgid_plural "Users" +msgstr[0] "" +msgstr[1] "" + +msgid "Permission Type" +msgstr "" + +msgid "Mode" +msgstr "" + +msgid "Resource" +msgstr "" + +msgid "Reason" +msgstr "" + +msgid "Short reasoning behind the request" +msgstr "" + +msgid "function performed by the responsible party" +msgstr "" + +msgid "brief narrative summary of the content of the resource(s)" +msgstr "" + +msgid "reference date for the cited resource" +msgstr "" + +msgid "identification of when a given event occurred" +msgstr "" + +msgid "version of the cited resource" +msgstr "" + +msgid "" +"authority or function assigned, as to a ruler, legislative assembly, " +"delegate, or the like." +msgstr "" + +msgid "a DOI will be added by Admin before publication." +msgstr "" + +msgid "summary of the intentions with which the resource(s) was developed" +msgstr "" + +msgid "" +"frequency with which modifications and deletions are made to the data after " +"it is first produced" +msgstr "" + +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated)" +msgstr "" + +msgid "" +"formalised word(s) or phrase(s) from a fixed thesaurus used to describe the " +"subject (space or comma-separated)" +msgstr "" + +msgid "keyword identifies a location" +msgstr "" + +msgid "limitation(s) placed upon the access or use of the data." +msgstr "" + +msgid "" +"other restrictions and legal prerequisites for accessing and using the " +"resource or metadata" +msgstr "" + +msgid "license of the dataset" +msgstr "" + +msgid "language used within the dataset" +msgstr "" + +msgid "" +"high-level geographic data thematic classification to assist in the grouping " +"and search of available geographic data sets." +msgstr "" + +msgid "method used to represent geographic information in the dataset." +msgstr "" + +msgid "time period covered by the content of the dataset (start)" +msgstr "" + +msgid "time period covered by the content of the dataset (end)" +msgstr "" + +msgid "" +"general explanation of the data producer's knowledge about the lineage of a " +"dataset" +msgstr "" + +msgid "title" +msgstr "" + +msgid "name by which the cited resource is known" +msgstr "" + +msgid "abstract" +msgstr "" + +msgid "purpose" +msgstr "" + +msgid "date" +msgstr "" + +msgid "date type" +msgstr "" + +msgid "edition" +msgstr "" + +msgid "Attribution" +msgstr "" + +msgid "DOI" +msgstr "" + +msgid "maintenance frequency" +msgstr "" + +msgid "keywords" +msgstr "" + +msgid "keywords region" +msgstr "" + +msgid "restrictions" +msgstr "" + +msgid "restrictions other" +msgstr "" + +msgid "language" +msgstr "" + +msgid "spatial representation type" +msgstr "" + +msgid "supplemental information" +msgstr "" + +msgid "any other descriptive information about the dataset" +msgstr "" + +msgid "data quality statement" +msgstr "" + +msgid "CSW typename" +msgstr "" + +msgid "CSW schema" +msgstr "" + +msgid "CSW source" +msgstr "" + +msgid "CSW insert date" +msgstr "" + +msgid "CSW type" +msgstr "" + +msgid "CSW anytext" +msgstr "" + +msgid "CSW WKT geometry" +msgstr "" + +msgid "Metadata uploaded preserve" +msgstr "" + +msgid "Featured" +msgstr "" + +msgid "Should this resource be advertised in home page?" +msgstr "" + +msgid "Is Published" +msgstr "" + +msgid "Should this resource be published and searchable?" +msgstr "" + +msgid "Approved" +msgstr "" + +msgid "Is this resource validated from a publisher or editor?" +msgstr "" + +msgid "Thumbnail url" +msgstr "" + +msgid "State" +msgstr "" + +msgid "Hold the resource processing state." +msgstr "" + +msgid "Dirty State" +msgstr "" + +msgid "Security Rules Are Not Synched with GeoServer!" +msgstr "" + +msgid "Resource Type" +msgstr "" + +msgid "Metadata" +msgstr "" + +msgid "If true, will be excluded from search" +msgstr "" + +msgid "For example \"kml\"" +msgstr "" + +msgid "For example \"View in Google Earth\"" +msgstr "" + +msgid "For example \"text/xml\"" +msgstr "" + +msgid "About" +msgstr "" + +msgid "Responsible, Point of Contact, Metadata Author" +msgstr "" + +msgid "Responsible" +msgstr "" + +msgid "Title" +msgstr "" + +msgid "SRID" +msgstr "" + +msgid "For more info see" +msgstr "" + +msgid "Creation Date" +msgstr "" + +msgid "Publication Date" +msgstr "" + +msgid "Revision Date" +msgstr "" + +msgid "Type" +msgstr "" + +msgid "Keywords" +msgstr "" + +msgid "More info" +msgstr "" + +msgid "Maintenance Frequency" +msgstr "" + +msgid "Restrictions" +msgstr "" + +msgid "Edition" +msgstr "" + +msgid "Temporal Extent" +msgstr "" + +msgid "Data Quality" +msgstr "" + +msgid "Supplemental Information" +msgstr "" + +msgid "Spatial Representation Type" +msgstr "" + +msgid "No content created yet." +msgstr "" + +msgid "Temporal Serie" +msgstr "" + +msgid "Select" +msgstr "" + +msgid "Service is" +msgstr "" + +msgid "online" +msgstr "" + +msgid "offline" +msgstr "" + +msgid "SECURITY NOT YET SYNCHRONIZED" +msgstr "" + +msgid "Sync permissions immediately" +msgstr "" + +msgid "PENDING APPROVAL" +msgstr "" + +msgid "UNPUBLISHED" +msgstr "" + +msgid "Create a Map" +msgstr "" + +msgid "View Map" +msgstr "" + +msgid "Batch Edit" +msgstr "" + +msgid "Cancel" +msgstr "" + +msgid "Submit" +msgstr "" + +msgid "Set Permissions" +msgstr "" + +msgid "Discover the available datasets." +msgstr "" + +msgid "Upload a Thumbnail" +msgstr "" + +msgid "Remove (and use auto generated thumbnail)" +msgstr "" + +msgid "Upload" +msgstr "" + +msgid "Auto generated thumbnail" +msgstr "" + +msgid "Raster Layer" +msgstr "" + +msgid "Vector Layer" +msgstr "" + +msgid "Vector Temporal Serie" +msgstr "" + +msgid "Remote Layer" +msgstr "" + +msgid "WMS Cascade Layer" +msgstr "" + +msgid "me" +msgstr "" + +msgid "System message: A request to modify resource" +msgstr "" + +msgid "The resource owner has requested to modify the resource" +msgstr "" + +msgid "Resource title" +msgstr "" + +msgid "Reason for the request" +msgstr "" + +msgid "" +"To allow the change, set the resource to not \"Approved\" under the metadata " +"settingsand write message to the owner to notify him" +msgstr "" + +msgid "Resource Cloned Successfully!" +msgstr "" + +#, python-brace-format +msgid "Error Occurred while Cloning the Resource: {e}" +msgstr "" + +msgid "Resource Metadata" +msgstr "" + +msgid "Thumbnail" +msgstr "" + +msgid "Resource ID" +msgstr "" + +msgid "Start" +msgstr "" + +msgid "End" +msgstr "" + +msgid "Extent" +msgstr "" + +msgid "Spatial Reference System Identifier" +msgstr "" + +msgid "Attributes" +msgstr "" + +msgid "Attribute name" +msgstr "" + +msgid "Description" +msgstr "" + +msgid "GeoNode Client Library" +msgstr "" + +msgid "" +"Flora and/or fauna in natural environment. Examples: wildlife, vegetation, " +"biological sciences, ecology, wilderness, sealife, wetlands, habitat." +msgstr "" + +msgid "Biota" +msgstr "" + +msgid "" +"Legal land descriptions. Examples: political and administrative boundaries." +msgstr "" + +msgid "Boundaries" +msgstr "" + +msgid "" +"Processes and phenomena of the atmosphere. Examples: cloud cover, weather, " +"climate, atmospheric conditions, climate change, precipitation." +msgstr "" + +msgid "Climatology Meteorology Atmosphere" +msgstr "" + +msgid "" +"Economic activities, conditions and employment. Examples: production, " +"labour, revenue, commerce, industry, tourism and ecotourism, forestry, " +"fisheries, commercial or subsistence hunting, exploration and exploitation " +"of resources such as minerals, oil and gas." +msgstr "" + +msgid "Economy" +msgstr "" + +msgid "" +"Height above or below sea level. Examples: altitude, bathymetry, digital " +"elevation models, slope, derived products." +msgstr "" + +msgid "Elevation" +msgstr "" + +msgid "" +"Environmental resources, protection and conservation. Examples: " +"environmental pollution, waste storage and treatment, environmental impact " +"assessment, monitoring environmental risk, nature reserves, landscape." +msgstr "" + +msgid "Environment" +msgstr "" + +msgid "" +"Rearing of animals and/or cultivation of plants. Examples: agriculture, " +"irrigation, aquaculture, plantations, herding, pests and diseases affecting " +"crops and livestock." +msgstr "" + +msgid "Farming" +msgstr "" + +msgid "" +"Information pertaining to earth sciences. Examples: geophysical features and " +"processes, geology, minerals, sciences dealing with the composition, " +"structure and origin of the earth s rocks, risks of earthquakes, volcanic " +"activity, landslides, gravity information, soils, permafrost, hydrogeology, " +"erosion." +msgstr "" + +msgid "Geoscientific Information" +msgstr "" + +msgid "" +"Health, health services, human ecology, and safety. Examples: disease and " +"illness, factors affecting health, hygiene, substance abuse, mental and " +"physical health, health services." +msgstr "" + +msgid "Health" +msgstr "" + +msgid "" +"Base maps. Examples: land cover, topographic maps, imagery, unclassified " +"images, annotations." +msgstr "" + +msgid "Imagery Base Maps Earth Cover" +msgstr "" + +msgid "" +"Inland water features, drainage systems and their characteristics. Examples: " +"rivers and glaciers, salt lakes, water utilization plans, dams, currents, " +"floods, water quality, hydrographic charts." +msgstr "" + +msgid "Inland Waters" +msgstr "" + +msgid "" +"Military bases, structures, activities. Examples: barracks, training " +"grounds, military transportation, information collection." +msgstr "" + +msgid "Intelligence Military" +msgstr "" + +msgid "" +"Positional information and services. Examples: addresses, geodetic networks, " +"control points, postal zones and services, place names." +msgstr "" + +msgid "Location" +msgstr "" + +msgid "" +"Features and characteristics of salt water bodies (excluding inland waters). " +"Examples: tides, tidal waves, coastal information, reefs." +msgstr "" + +msgid "Oceans" +msgstr "" + +msgid "" +"Information used for appropriate actions for future use of the land. " +"Examples: land use maps, zoning maps, cadastral surveys, land ownership." +msgstr "" + +msgid "Planning Cadastre" +msgstr "" + +msgid "" +"Settlements, anthropology, archaeology, education, traditional beliefs, " +"manners and customs, demographic data, recreational areas and activities, " +"social impact assessments, crime and justice, census information. Economic " +"activities, conditions and employment." +msgstr "" + +msgid "Population" +msgstr "" + +msgid "" +"Characteristics of society and cultures. Examples: settlements, " +"anthropology, archaeology, education, traditional beliefs, manners and " +"customs, demographic data, recreational areas and activities, social impact " +"assessments, crime and justice, census information." +msgstr "" + +msgid "Society" +msgstr "" + +msgid "" +"Man-made construction. Examples: buildings, museums, churches, factories, " +"housing, monuments, shops, towers." +msgstr "" + +msgid "Structure" +msgstr "" + +msgid "" +"Means and aids for conveying persons and/or goods. Examples: roads, airports/" +"airstrips, shipping routes, tunnels, nautical charts, vehicle or vessel " +"location, aeronautical charts, railways." +msgstr "" + +msgid "Transportation" +msgstr "" + +msgid "" +"Energy, water and waste systems and communications infrastructure and " +"services. Examples: hydroelectricity, geothermal, solar and nuclear sources " +"of energy, water purification and distribution, sewage collection and " +"disposal, electricity and gas distribution, data communication, " +"telecommunication, radio, communication networks." +msgstr "" + +msgid "Utilities Communication" +msgstr "" + +msgid "Document Created" +msgstr "" + +msgid "A Document was created" +msgstr "" + +msgid "Document Updated" +msgstr "" + +msgid "A Document was updated" +msgstr "" + +msgid "Document Approved" +msgstr "" + +msgid "A Document was approved by a Manager" +msgstr "" + +msgid "Document Published" +msgstr "" + +msgid "A Document was published" +msgstr "" + +msgid "Document Deleted" +msgstr "" + +msgid "A Document was deleted" +msgstr "" + +msgid "Comment on Document" +msgstr "" + +msgid "A Document was commented on" +msgstr "" + +msgid "Rating for Document" +msgstr "" + +msgid "A rating was given to a document" +msgstr "" + +msgid "Width" +msgstr "" + +msgid "Height" +msgstr "" + +msgid "Make" +msgstr "" + +msgid "Model" +msgstr "" + +msgid "Latitude" +msgstr "" + +msgid "Not specified" +msgstr "" + +msgid "Longitude" +msgstr "" + +msgid "Flash" +msgstr "" + +msgid "True" +msgstr "" + +msgid "False" +msgstr "" + +msgid "Speed Rating" +msgstr "" + +msgid "Link to" +msgstr "" + +msgid "File" +msgstr "" + +msgid "Document must be a file or url." +msgstr "" + +msgid "A document cannot have both a file and a url." +msgstr "" + +msgid "This file type is not allowed" +msgstr "" + +msgid "Permissions must be valid JSON." +msgstr "" + +msgid "The URL of the document if it is external." +msgstr "" + +msgid "URL" +msgstr "" + +msgid "Your browser does not support the audio element." +msgstr "" + +msgid "Your browser does not support the video tag." +msgstr "" + +msgid "Download the" +msgstr "" + +msgid "document" +msgstr "" + +msgid "External Resource" +msgstr "" + +msgid "Rate this document" +msgstr "" + +msgid "Average Rating" +msgstr "" + +msgid "Metadata Detail" +msgstr "" + +msgid "Download Document" +msgstr "" + +msgid "Request Download" +msgstr "" + +msgid "Edit Document" +msgstr "" + +msgid "Request change" +msgstr "" + +msgid "Wizard" +msgstr "" + +msgid "Advanced Edit" +msgstr "" + +msgid "Document" +msgid_plural "Documents" +msgstr[0] "" +msgstr[1] "" + +msgid "Clone" +msgstr "" + +msgid "Replace" +msgstr "" + +msgid "Remove" +msgstr "" + +msgid "Close" +msgstr "" + +msgid "Download Metadata" +msgstr "" + +msgid "Full metadata" +msgstr "" + +msgid "Text format" +msgstr "" + +msgid "HTML format" +msgstr "" + +msgid "Standard Metadata - XML format" +msgstr "" + +msgid "Resources using this document" +msgstr "" + +msgid "List of resources using this document:" +msgstr "" + +msgid "This document is not related to any maps or layers" +msgstr "" + +msgid "Permissions" +msgstr "" + +msgid "Click the button below to change the permissions of this document." +msgstr "" + +msgid "Change Document Permissions" +msgstr "" + +msgid "Explore Documents" +msgstr "" + +msgid "Upload Documents" +msgstr "" + +#, python-format +msgid "for %(document_title)s" +msgstr "" + +msgid "Completeness" +msgstr "" + +msgid "Metadata Schema mandatory fields completed" +msgstr "" + +msgid "Check Schema mandatory fields" +msgstr "" + +msgid "Error updating metadata. Please check the following fields: " +msgstr "" + +msgid "Done" +msgstr "" + +msgid "Metadata Provider" +msgstr "" + +msgid "<< Back" +msgstr "" + +msgid "Update" +msgstr "" + +msgid "Next >>" +msgstr "" + +msgid "Select an option" +msgstr "" + +msgid "Edit Metadata" +msgstr "" + +msgid "Editing details for" +msgstr "" + +msgid "Return to Document" +msgstr "" + +msgid "Deleting" +msgstr "" + +msgid "Remove Document" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(document_title)s?\n" +" " +msgstr "" + +msgid "Yes, I am sure" +msgstr "" + +msgid "No, don't remove it" +msgstr "" + +msgid "Replace Document" +msgstr "" + +msgid "Replace " +msgstr "" + +msgid "Clear" +msgstr "" + +msgid "Upload Document's Thumbnail" +msgstr "" + +msgid "Drop files here" +msgstr "" + +msgid " or select them one by one:" +msgstr "" + +msgid "Choose Files" +msgstr "" + +msgid "Files to be uploaded" +msgstr "" + +msgid "Is Upload Metadata XML Form" +msgstr "" + +msgid "Upload files" +msgstr "" + +msgid "Upload Document" +msgstr "" + +msgid "Allowed document types" +msgstr "" + +msgid "Done!" +msgstr "" + +msgid "Edit" +msgstr "" + +msgid "Settings" +msgstr "" + +msgid "Mandatory" +msgstr "" + +msgid "Optional" +msgstr "" + +msgid "Basic Metadata" +msgstr "" + +msgid "Location and Licenses" +msgstr "" + +msgid "Required Metadata" +msgstr "" + +msgid "Optional Metadata" +msgstr "" + +msgid "Choose one of the following..." +msgstr "" + +msgid "Other, Optional, Metadata" +msgstr "" + +msgid "Responsible Parties" +msgstr "" + +msgid "Responsible and Permissions" +msgstr "" + +msgid "Publishing" +msgstr "" + +msgid "You are not allowed to view this document." +msgstr "" + +msgid "You are not permitted to delete this document" +msgstr "" + +msgid "You do not have permissions for this document." +msgstr "" + +msgid "You are not permitted to modify this document" +msgstr "" + +msgid "You are not permitted to modify this document's metadata" +msgstr "" + +msgid "You are not permitted to view this document" +msgstr "" + +msgid "Not allowed" +msgstr "" + +msgid "Not found" +msgstr "" + +msgid "You must set a point of contact for this resource" +msgstr "" + +msgid "You must set an author for this resource" +msgstr "" + +msgid "Favorite" +msgstr "" + +msgid "Add to Favorites" +msgstr "" + +msgid "Go to Favorites" +msgstr "" + +msgid "Log in to add/delete Favorites." +msgstr "" + +msgid "Delete from Favorites" +msgstr "" + +msgid "Favorites" +msgstr "" + +msgid "Favorites for" +msgstr "" + +msgid "Item" +msgstr "" + +msgid "App Created" +msgstr "" + +msgid "A App was created" +msgstr "" + +msgid "App Updated" +msgstr "" + +msgid "A App was updated" +msgstr "" + +msgid "App Approved" +msgstr "" + +msgid "A App was approved by a Manager" +msgstr "" + +msgid "App Published" +msgstr "" + +msgid "A App was published" +msgstr "" + +msgid "App Deleted" +msgstr "" + +msgid "A App was deleted" +msgstr "" + +msgid "Comment on App" +msgstr "" + +msgid "An App was commented on" +msgstr "" + +msgid "Rating for App" +msgstr "" + +msgid "A rating was given to an App" +msgstr "" + +msgid "Name" +msgstr "" + +msgid "zoom" +msgstr "" + +msgid "projection" +msgstr "" + +msgid "center X" +msgstr "" + +msgid "center Y" +msgstr "" + +msgid "Site URL" +msgstr "" + +#, python-format +msgid "%(resource.type)s" +msgstr "" + +msgid "Rate this" +msgstr "" + +msgid "Editing Tools" +msgstr "" + +msgid "Set" +msgstr "" + +msgid "View" +msgstr "" + +msgid "Specify which users can view or modify this map" +msgstr "" + +msgid "Change Permissions of this" +msgstr "" + +msgid "Embed Iframe" +msgstr "" + +msgid "" +"To embed this map, add the following code snippet and customize its " +"properties (scrolling, width, height) based on your needs to your site" +msgstr "" + +msgid "Download" +msgstr "" + +#, python-format +msgid "%(GEONODE_APPS_NAME)s" +msgstr "" + +msgid "Create New" +msgstr "" + +msgid "Explore" +msgstr "" + +#, python-format +msgid "for %(map_title)s" +msgstr "" + +msgid "" +"Note: this geoapp's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" + +msgid "Explore Apps" +msgstr "" + +#, python-format +msgid "" +"\n" +" Editing details for %(map_title)s\n" +" " +msgstr "" + +msgid "Return to GeoApp" +msgstr "" + +msgid "New Map" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(resource_title)s?\n" +" " +msgstr "" + +msgid "Other Settings" +msgstr "" + +msgid "You are not permitted to delete this app." +msgstr "" + +msgid "You do not have permissions for this app." +msgstr "" + +msgid "You must be logged in to save this app" +msgstr "" + +msgid "You are not permitted to save or edit this app." +msgstr "" + +msgid "You are not allowed to modify this app's metadata." +msgstr "" + +msgid "You are not allowed to view this app." +msgstr "" + +msgid "An unknown error has occured." +msgstr "" + +msgid "Layer Uploaded" +msgstr "" + +msgid "A layer was uploaded" +msgstr "" + +msgid "Comment on Layer" +msgstr "" + +msgid "A layer was commented on" +msgstr "" + +msgid "Rating for Layer" +msgstr "" + +msgid "A rating was given to a layer" +msgstr "" + +msgid "Points" +msgstr "" + +msgid "Lines" +msgstr "" + +msgid "Polygons" +msgstr "" + +msgid "Dataset name" +msgstr "" + +msgid "Dataset title" +msgstr "" + +msgid "Geometry type" +msgstr "" + +msgid "Create Dataset" +msgstr "" + +msgid "Explore Datasets" +msgstr "" + +msgid "Create an empty dataset" +msgstr "" + +msgid "Add Attribute" +msgstr "" + +msgid "Create" +msgstr "" + +msgid "String" +msgstr "" + +msgid "Integer" +msgstr "" + +msgid "Float" +msgstr "" + +msgid "No title provided" +msgstr "" + +msgid "No abstract provided" +msgstr "" + +msgid "An exception occurred loading data to PostGIS" +msgstr "" + +msgid " Additionally an error occured during database cleanup" +msgstr "" + +msgid "Invalid Projection. Layer is missing CRS!" +msgstr "" + +msgid "GZIP" +msgstr "" + +msgid "GeoTIFF" +msgstr "" + +msgid "Zipped Shapefile" +msgstr "" + +msgid "GML 2.0" +msgstr "" + +msgid "GML 3.1.1" +msgstr "" + +msgid "CSV" +msgstr "" + +msgid "Excel" +msgstr "" + +msgid "GeoJSON" +msgstr "" + +msgid "JPEG" +msgstr "" + +msgid "PDF" +msgstr "" + +msgid "PNG" +msgstr "" + +msgid "You don't have permissions to change style for this layer" +msgstr "" + +msgid "Bad HTTP Authorization Credentials." +msgstr "" + +msgid "" +"a short version of the name consisting only of letters, numbers, underscores " +"and hyphens." +msgstr "" + +msgid "A group already exists with that slug." +msgstr "" + +msgid "A group already exists with that name." +msgstr "" + +msgid "User Identifiers" +msgstr "" + +msgid "Assign manager role" +msgstr "" + +#, python-format +msgid "" +"The following are not valid usernames: %(errors)s; not added to the group" +msgstr "" + +msgid "Group Categories" +msgstr "" + +msgid "Public" +msgstr "" + +msgid "Public (invite-only)" +msgstr "" + +msgid "Private" +msgstr "" + +msgid "" +"Public: Any registered user can view and join a public group.
          Public " +"(invite-only):Any registered user can view the group. Only invited users " +"can join.
          Private: Registered users cannot see any details about the " +"group, including membership. Only invited users can join." +msgstr "" + +msgid "" +"Email used to contact one or all group members, such as a mailing list, " +"shared email, or exchange group." +msgstr "" + +msgid "Logo" +msgstr "" + +msgid "Email" +msgstr "" + +msgid "A space or comma-separated list of keywords" +msgstr "" + +msgid "Access" +msgstr "" + +msgid "Categories" +msgstr "" + +msgid "Manager" +msgstr "" + +msgid "Member" +msgstr "" + +msgid "Activity Feed for" +msgstr "" + +msgid "All" +msgstr "" + +msgid "Layers" +msgstr "" + +msgid "Maps" +msgstr "" + +msgid "Documents" +msgstr "" + +msgid "Comments" +msgstr "" + +msgid "No actions yet" +msgstr "" + +msgid "Explore Group Categories" +msgstr "" + +msgid "Create a New Group Category" +msgstr "" + +msgid "Explore Groups" +msgstr "" + +msgid "Create a Group" +msgstr "" + +msgid "Update Group" +msgstr "" + +msgid "Remove Group" +msgstr "" + +msgid "Create Group" +msgstr "" + +msgid "Last Modified" +msgstr "" + +msgid "This group has not created a logo." +msgstr "" + +msgid "Edit Group Details" +msgstr "" + +msgid "Manage Group Members" +msgstr "" + +msgid "Delete this Group" +msgstr "" + +msgid "Group Activities" +msgstr "" + +#, python-format +msgid "" +"\n" +" This group is %(access)s.\n" +" " +msgstr "" + +msgid "Anyone may join this group." +msgstr "" + +msgid "Join Group" +msgstr "" + +msgid "Anyone may view this group but membership is by invitation only." +msgstr "" + +msgid "Membership is by invitation only." +msgstr "" + +msgid "Managers" +msgstr "" + +msgid "Members" +msgstr "" + +msgid "Create a New Group" +msgstr "" + +msgid "Edit Members for" +msgstr "" + +msgid "Current Members" +msgstr "" + +msgid "Promote" +msgstr "" + +msgid "Demote" +msgstr "" + +msgid "Role" +msgstr "" + +msgid "Add new members" +msgstr "" + +msgid "Add Group Members" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove the group: %(group_title)s?\n" +" " +msgstr "" + +msgid "Remove Maps/Data from Group" +msgstr "" + +msgid "Remove selected maps/data" +msgstr "" + +msgid "Edit Group Category Details" +msgstr "" + +msgid "Groups" +msgstr "" + +msgid "Create Group Category" +msgstr "" + +msgid "Create a Group Category" +msgstr "" + +msgid "Edit Group Category" +msgstr "" + +msgid "Save" +msgstr "" + +msgid "ready" +msgstr "" + +msgid "updating-harvestable-resources" +msgstr "" + +msgid "harvesting-resources" +msgstr "" + +msgid "checking-availability" +msgstr "" + +msgid "Harvester name" +msgstr "" + +msgid "Base URL of the remote service that is to be harvested" +msgstr "" + +msgid "" +"Whether to periodically schedule this harvester to look for resources on the " +"remote service" +msgstr "" + +msgid "" +"How often (in minutes) should new harvesting sessions be automatically " +"scheduled? Setting this value to zero has the same effect as setting " +"`scheduling_enabled` to False " +msgstr "" + +msgid "Whether the remote service is known to be available or not" +msgstr "" + +msgid "" +"How often (in minutes) should the remote service be checked for availability?" +msgstr "" + +msgid "Last time the remote server was checked for availability" +msgstr "" + +msgid "Last time the remote server was checked for harvestable resources" +msgstr "" + +msgid "Default owner of harvested resources" +msgstr "" + +msgid "Default access permissions of harvested resources" +msgstr "" + +msgid "" +"Should new resources be harvested automatically without explicit selection?" +msgstr "" + +msgid "" +"Orphan resources are those that have previously been created by means of a " +"harvesting operation but that GeoNode can no longer find on the remote " +"service being harvested. Should these resources be deleted from GeoNode " +"automatically? This also applies to when a harvester configuration is " +"deleted, in which case all of the resources that originated from that " +"harvester are now considered to be orphan." +msgstr "" + +msgid "Date of last update to the harvester configuration." +msgstr "" + +msgid "" +"Harvester class used to perform harvesting sessions. New harvester types can " +"be added by an admin by changing the main GeoNode `settings.py` file" +msgstr "" + +msgid "" +"Configuration specific to each harvester type. Please consult GeoNode " +"documentation on harvesting for more info. This field is mandatory, so at " +"the very least an empty object (i.e. {}) must be supplied." +msgstr "" + +msgid "Periodic task used to configure harvest scheduling" +msgstr "" + +msgid "Periodic task used to check availability of the remote" +msgstr "" + +#, python-brace-format +msgid "Check availability of {self.name}" +msgstr "" + +msgid "updating-harvestable-resource" +msgstr "" + +msgid "harvesting-resource" +msgstr "" + +msgid "" +"Identifier that allows referencing the resource on its remote service in a " +"unique fashion. This is usually automatically filled by the harvester " +"worker. The harvester worker needs to know how to either read or generate " +"this from each remote resource in order to be able to compare the " +"availability of resources between consecutive harvesting sessions." +msgstr "" + +msgid "" +"Type of the resource in the remote service. Each harvester worker knows how " +"to fill this field, in accordance with the resources for which harvesting is " +"supported" +msgstr "" + +#, python-brace-format +msgid "The e-mail address '{email}' has already been invited." +msgstr "" + +#, python-brace-format +msgid "The e-mail address '{email}' has already accepted an invite." +msgstr "" + +#, python-brace-format +msgid "An active user is already using the e-mail address '{email}'" +msgstr "" + +msgid "E-mail" +msgstr "" + +#, python-format +msgid "Invitations successfully sent to '%(email)s'" +msgstr "" + +#, python-format +msgid "" +"Sorry, it was not possible to invite '%(email)s' due to the following isse: " +"%(error)s (%(type)s)" +msgstr "" + +msgid "Layer Created" +msgstr "" + +msgid "A Layer was created" +msgstr "" + +msgid "Layer Updated" +msgstr "" + +msgid "A Layer was updated" +msgstr "" + +msgid "Layer Approved" +msgstr "" + +msgid "A Layer was approved by a Manager" +msgstr "" + +msgid "Layer Published" +msgstr "" + +msgid "A Layer was published" +msgstr "" + +msgid "Layer Deleted" +msgstr "" + +msgid "A Layer was deleted" +msgstr "" + +msgid "YYYYMMDD" +msgstr "" + +msgid "YYYYMMDD'T'hhmmss" +msgstr "" + +msgid "YYYYMMDD'T'hhmmss'Z'" +msgstr "" + +msgid "style name" +msgstr "" + +msgid "sld text" +msgstr "" + +msgid "sld version" +msgstr "" + +msgid "sld url" +msgstr "" + +msgid "Workspace" +msgstr "" + +msgid "Store" +msgstr "" + +msgid "Storetype" +msgstr "" + +msgid "Typename" +msgstr "" + +msgid "Is mosaic?" +msgstr "" + +msgid "Has time?" +msgstr "" + +msgid "Has elevation?" +msgstr "" + +msgid "Time regex" +msgstr "" + +msgid "Elevation regex" +msgstr "" + +msgid "use featureinfo custom template?" +msgstr "" + +msgid "specifies wether or not use a custom GetFeatureInfo template." +msgstr "" + +msgid "featureinfo custom template" +msgstr "" + +msgid "the custom GetFeatureInfo template HTML contents." +msgstr "" + +msgid "File cannot be opened, maybe check the encoding" +msgstr "" + +msgid "attribute name" +msgstr "" + +msgid "name of attribute as stored in shapefile/spatial database" +msgstr "" + +msgid "attribute description" +msgstr "" + +msgid "description of attribute to be used in metadata" +msgstr "" + +msgid "attribute label" +msgstr "" + +msgid "title of attribute as displayed in GeoNode" +msgstr "" + +msgid "attribute type" +msgstr "" + +msgid "the data type of the attribute (integer, string, geometry, etc)" +msgstr "" + +msgid "visible?" +msgstr "" + +msgid "specifies if the attribute should be displayed in identify results" +msgstr "" + +msgid "display order" +msgstr "" + +msgid "" +"specifies the order in which attribute should be displayed in identify " +"results" +msgstr "" + +msgid "Label" +msgstr "" + +msgid "Image" +msgstr "" + +msgid "Video (mp4)" +msgstr "" + +msgid "Video (ogg)" +msgstr "" + +msgid "Video (webm)" +msgstr "" + +msgid "Video (3gp)" +msgstr "" + +msgid "Video (flv)" +msgstr "" + +msgid "Video (YouTube/VIMEO - embedded)" +msgstr "" + +msgid "Audio" +msgstr "" + +msgid "IFRAME" +msgstr "" + +msgid "featureinfo type" +msgstr "" + +msgid "" +"specifies if the attribute should be rendered with an HTML widget on " +"GetFeatureInfo template." +msgstr "" + +msgid "count" +msgstr "" + +msgid "count value for this field" +msgstr "" + +msgid "min" +msgstr "" + +msgid "minimum value for this field" +msgstr "" + +msgid "max" +msgstr "" + +msgid "maximum value for this field" +msgstr "" + +msgid "average" +msgstr "" + +msgid "average value for this field" +msgstr "" + +msgid "median" +msgstr "" + +msgid "median value for this field" +msgstr "" + +msgid "standard deviation" +msgstr "" + +msgid "standard deviation for this field" +msgstr "" + +msgid "sum" +msgstr "" + +msgid "sum value for this field" +msgstr "" + +msgid "unique values for this field" +msgstr "" + +msgid "last modified" +msgstr "" + +msgid "date when attribute statistics were last updated" +msgstr "" + +msgid "Append to Dataset" +msgstr "" + +msgid "Return to Dataset" +msgstr "" + +msgid "Preserve Metadata XML" +msgstr "" + +msgid "Select the charset or leave default" +msgstr "" + +msgid "Home" +msgstr "" + +msgid "Change point of contact" +msgstr "" + +msgid "Assign a new point of contact to the layers below:" +msgstr "" + +msgid "Dataset WMS GetCapabilities document" +msgstr "" + +msgid "Filter Granules" +msgstr "" + +msgid "Active Filter:" +msgstr "" + +msgid "Granule ID" +msgstr "" + +msgid "Bounding Box" +msgstr "" + +msgid "Time" +msgstr "" + +msgid "Actions" +msgstr "" + +msgid "Granule Preview" +msgstr "" + +msgid "Granule Remove" +msgstr "" + +msgid "Dimension" +msgstr "" + +msgid "Enabled" +msgstr "" + +msgid "Regex" +msgstr "" + +msgid "TIME" +msgstr "" + +msgid "ELEVATION" +msgstr "" + +msgid "Attribute Name" +msgstr "" + +msgid "Range" +msgstr "" + +msgid "Average" +msgstr "" + +msgid "Median" +msgstr "" + +msgid "Standard Deviation" +msgstr "" + +msgid "Rate this dataset" +msgstr "" + +msgid "Analyze with" +msgstr "" + +msgid "Download Dataset" +msgstr "" + +msgid "Images" +msgstr "" + +msgid "Data" +msgstr "" + +msgid "Click to filter the layer" +msgstr "" + +msgid "Do you want to filter it?" +msgstr "" + +msgid "Loading..." +msgstr "" + +msgid "Filter by attributes" +msgstr "" + +msgid "Match" +msgstr "" + +msgid "all" +msgstr "" + +msgid "any" +msgstr "" + +msgid "of the following:" +msgstr "" + +msgid "Attribute" +msgstr "" + +msgid "Operator" +msgstr "" + +msgid "Feature limit" +msgstr "" + +msgid "Pick your download format:" +msgstr "" + +msgid "" +"No data available for this resource. Please contact a system administrator " +"or a manager." +msgstr "" + +msgid "Warning" +msgstr "" + +msgid "Edit Layer" +msgstr "" + +msgid "Upload Metadata" +msgstr "" + +msgid "Styles" +msgstr "" + +msgid "Manage" +msgstr "" + +msgid "Layer" +msgid_plural "Layers" +msgstr[0] "" +msgstr[1] "" + +msgid "Append Data" +msgstr "" + +msgid "Edit data" +msgstr "" + +msgid "View Dataset" +msgstr "" + +msgid "Attribute Information" +msgstr "" + +msgid "ISO Feature Catalogue" +msgstr "" + +msgid "Legend" +msgstr "" + +msgid "Maps using this dataset" +msgstr "" + +msgid "List of maps using this dataset:" +msgstr "" + +msgid "This dataset is not currently used in any maps." +msgstr "" + +msgid "Create a map using this dataset" +msgstr "" + +msgid "Click the button below to generate a new map based on this dataset." +msgstr "" + +msgid "Add the dataset to an existing map" +msgstr "" + +msgid "Click the button below to add the dataset to the selected map." +msgstr "" + +msgid "Add to Map" +msgstr "" + +msgid "Documents related to this layer" +msgstr "" + +msgid "List of documents related to this layer:" +msgstr "" + +msgid "" +"The following styles are associated with this layer. Choose a style to view " +"it in the preview map." +msgstr "" + +msgid "(default style)" +msgstr "" + +msgid "No styles associated with this layer" +msgstr "" + +msgid "External service layer" +msgstr "" + +msgid "Source" +msgstr "" + +msgid "Refresh Attributes and Statistics of this layer" +msgstr "" + +msgid "" +"Click the button below to allow GeoNode refreshing the list of available " +"Dataset Attributes. If the option 'WPS_ENABLED' has been also set on the " +"backend, it will recalculate their statistics too." +msgstr "" + +msgid "Refresh Attributes and Statistics" +msgstr "" + +msgid "Clear the Server Cache of this layer" +msgstr "" + +msgid "Click the button below to wipe the tile-cache of this layer." +msgstr "" + +msgid "Empty Tiled-Dataset Cache" +msgstr "" + +msgid "Click the button below to change the permissions of this layer." +msgstr "" + +msgid "Change Dataset Permissions" +msgstr "" + +msgid "Remove Mosaic Granules" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove Granule %(granule_id)s of the " +"Mosaic %(layer_title)s?\n" +" " +msgstr "" + +msgid "This action affects the following maps:" +msgstr "" + +msgid "No maps are using this layer" +msgstr "" + +msgid "Upload Datasets" +msgstr "" + +#, python-format +msgid "for %(layer_title)s" +msgstr "" + +msgid "" +"Note: this layer's orginal metadata was populated and preserved by importing " +"a metadata XML file.\n" +" This metadata cannot be edited." +msgstr "" + +msgid "" +"Note: this layer's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and " +"Dublin Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" + +#, python-format +msgid "" +"\n" +" Editing details for %(layer_title)s\n" +" " +msgstr "" + +msgid "Use a custom template?" +msgstr "" + +msgid "Display Order" +msgstr "" + +msgid "Display Type" +msgstr "" + +msgid "Visible" +msgstr "" + +msgid "ERROR" +msgstr "" + +msgid "Following items are required:" +msgstr "" + +msgid "Upload Dataset Metadata" +msgstr "" + +msgid "(XML - ISO, FGDC, ebRIM, Dublin Core)" +msgstr "" + +msgid "Remove Datasets" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(layer_title)s?\n" +" " +msgstr "" + +msgid "Replace Dataset" +msgstr "" + +msgid "Manage Styles" +msgstr "" + +#, python-format +msgid "" +"\n" +" Manage Available Styles for " +"%(layer_title)s\n" +" " +msgstr "" + +msgid "Dataset Default Style" +msgstr "" + +msgid "Available styles" +msgstr "" + +msgid "" +"Click on an available style in the upper box to assign it to this layer. " +"Selected styles appear in the lower box." +msgstr "" + +msgid "Update Available Styles" +msgstr "" + +msgid "Return to Layer" +msgstr "" + +msgid "Upload Dataset Style" +msgstr "" + +msgid "(SLD - Style Layer Descriptor 1.0, 1.1)" +msgstr "" + +msgid "WARNING" +msgstr "" + +msgid "This will most probably overwrite the current default style!" +msgstr "" + +msgid "Preview" +msgstr "" + +msgid "Dataset Attributes" +msgstr "" + +msgid "Upload Dataset" +msgstr "" + +msgid "Upload status" +msgstr "" + +msgid "" +"This file needs additional configuration to complete the upload process. " +"Please click on the button to fill the required configuration" +msgstr "" + +msgid "Delete" +msgstr "" + +msgid "Upload process completed" +msgstr "" + +msgid "" +"The upload process cannot be completed because the original file is no more " +"available" +msgstr "" + +msgid "Created" +msgstr "" + +msgid "Progress" +msgstr "" + +msgid "page" +msgstr "" + +msgid "of" +msgstr "" + +msgid "Delete incomplete upload" +msgstr "" + +msgid "Are you sure you want to remove this incomplete upload?" +msgstr "" + +msgid "Upload Dataset Step: Set SRS" +msgstr "" + +msgid "Provide CRS for " +msgstr "" + +msgid "Coordinate Reference System" +msgstr "" + +msgid "" +"\n" +" A coordinate reference system for this dataset could not be " +"determined.\n" +" Locate or enter the appropriate ESPG code for this dataset " +"below.\n" +" One way to do this is do visit:\n" +" prj2epsg\n" +" and enter the following:\n" +" " +msgstr "" + +msgid "Native CRS could not be found!" +msgstr "" + +msgid "Provide EPSG code for Source SRS" +msgstr "" + +msgid "EPSG Code (Source SRS)" +msgstr "" + +msgid "Select a Source SRS" +msgstr "" + +msgid "" +"Source SRS EPSG Code is mandatory and represents the native data Spatial " +"Reference System.\n" +"

          \n" +" This must be coherent with the Geometry values (lon/lat " +"coordinates as an instance) stored on the geospatial dataset.\n" +"

          \n" +" If not specified on the geospatial data itself, it must " +"be manually declared by the operator.\n" +"

          \n" +" More information is provided at the bottom of the page " +"in the \"Additional Help\" sections.\n" +" " +msgstr "" + +msgid "Next" +msgstr "" + +msgid "Advanced Options" +msgstr "" + +msgid "Target SRS" +msgstr "" + +msgid "EPSG Code (Target SRS)" +msgstr "" + +msgid "Select a Target SRS" +msgstr "" + +msgid "leave it empty to use only Source SRS" +msgstr "" + +msgid "Additional Help" +msgstr "" + +msgid "Spatial Reference System" +msgstr "" + +msgid "" +"A spatial reference system (SRS) or coordinate reference system (CRS) is a " +"coordinate-based local,\n" +" regional or global system used to locate " +"geographical entities. A spatial reference system defines a specific map\n" +" projection, as well as transformations between " +"different spatial reference systems. Spatial reference systems are\n" +" defined by the OGC's Simple feature access using " +"well-known text, and support has been implemented by several\n" +" standards-based geographic information systems. " +"Spatial reference systems can be referred to using a SRID integer,\n" +" including EPSG codes defined by the " +"International Association of Oil and Gas Producers.\n" +" It is specified in ISO 19111:2007 Geographic " +"information—Spatial referencing by coordinates, also published as\n" +" OGC Abstract Specification, Topic 2: Spatial " +"referencing by coordinate." +msgstr "" + +msgid "Identifiers" +msgstr "" + +msgid "" +"\n" +" A Spatial Reference System Identifier (SRID) is a " +"unique value used to unambiguously identify projected, unprojected,\n" +" and local spatial coordinate system definitions. " +"These coordinate systems form the heart of all GIS applications.\n" +"\n" +" Virtually all major spatial vendors have created " +"their own SRID implementation or refer to those of an authority,\n" +" such as the European Petroleum Survey Group (EPSG).\n" +" " +msgstr "" + +msgid "" +"NOTE: As of 2005 the EPSG SRID values are now maintained by the " +"International\n" +" Association of Oil & Gas Producers (OGP) " +"Surveying & Positioning Committee" +msgstr "" + +msgid "" +"\n" +" SRIDs are the primary key for the Open Geospatial " +"Consortium (OGC) spatial_ref_sys metadata table for the Simple\n" +" Features for SQL Specification, Versions 1.1 and " +"1.2, which is defined as follows:\n" +" " +msgstr "" + +msgid "" +"\n" +" In spatially enabled databases (such as IBM DB2, IBM " +"Informix, Microsoft SQL Server, MySQL, Oracle RDBMS, Teradata, PostGIS and\n" +" SQL Anywhere), SRIDs are used to uniquely identify " +"the coordinate systems used to define columns of spatial data or individual\n" +" spatial objects in a spatial column (depending on " +"the spatial implementation). SRIDs are typically associated with a well " +"known\n" +" text (WKT) string definition of the coordinate " +"system (SRTEXT, above). From the Well Known Text Wikipedia page\n" +" " +msgstr "" + +msgid "" +"“A WKT string for a spatial reference system describes the datum, geoid, " +"coordinate system,\n" +" and map projection of the spatial objects”." +msgstr "" + +msgid "" +"\n" +" Here are two common coordinate systems with their " +"EPSG SRID value followed by their well known text:\n" +" " +msgstr "" + +msgid "" +"\n" +" UTM, Zone 17N, NAD27 — SRID 2029\n" +" " +msgstr "" + +msgid "" +"\n" +" WGS84 — SRID 4326\n" +" " +msgstr "" + +msgid "" +"\n" +" SRID values associated with spatial data can be used " +"to constrain spatial operations — for instance, spatial operations cannot be " +"performed\n" +" between spatial objects with differing SRIDs in some " +"systems, or trigger coordinate system transformations between spatial " +"objects in others.\n" +" " +msgstr "" + +msgid "" +"Source SRS EPSG Code is mandatory and represents the native data Spatial " +"Reference System. This must be coherent with the\n" +" Geometry values (lon/lat coordinates as an " +"instance) stored on the geospatial dataset. If not specified on the " +"geospatial data itself, it\n" +" must be manually declared by the operator." +msgstr "" + +msgid "" +"Target SRS EPSG Code is optional. This must be used only if we need to re-" +"project the coordinates from Source SRS to another one.\n" +" " +msgstr "" + +msgid "Upload Dataset Step: CSV Field Mapping" +msgstr "" + +msgid "Geospatial Data" +msgstr "" + +msgid "" +"Please indicate which attributes contain the latitude and longitude " +"coordinates in the CSV data." +msgstr "" + +msgid "" +"With this data, GeoNode was able to guess which attributes contain the\n" +" latitude and longitude coordinates, but please confirm that " +"the correct\n" +" attributes are selected below." +msgstr "" + +msgid "Select an attribute" +msgstr "" + +msgid "" +"We did not detect columns that could be used for the latitude and " +"longitude.\n" +" Please verify that you have two columns in your csv file that can be " +"used for\n" +" the latitude and longitude." +msgstr "" + +msgid "Your upload is either complete or you haven't resumed an earlier one." +msgstr "" + +msgid "Return to" +msgstr "" + +msgid "Upload Form" +msgstr "" + +msgid "Upload Dataset Step: Time" +msgstr "" + +msgid "Inspect data for " +msgstr "" + +msgid "Configure as Time-Series" +msgstr "" + +msgid "" +"Toggling this selector allows you to configure (or not) this data as a time " +"series; in this case you will also have to select an attribute\n" +" to drive the time dimension.\n" +"

          \n" +" If GeoNode is not able to parse any of the values for " +"the selected attribute red markers will appear to highlight the problems.\n" +"

          \n" +" More information is provided at the bottom of the page " +"in the \"Additional Help\" sections.\n" +" " +msgstr "" + +msgid "No" +msgstr "" + +msgid "Use an existing timestamp attribute in the data" +msgstr "" + +msgid "Yes: with an existing Time-Attribute" +msgstr "" + +msgid "" +"Yes: by converting data to a timestamp using standard date/time " +"representation" +msgstr "" + +msgid "Convert a number field into a year" +msgstr "" + +msgid "Yes: by converting a number as Year" +msgstr "" + +msgid "" +"Convert data to a timestamp using standard date/time representation or a " +"custom format" +msgstr "" + +msgid "Start Importer" +msgstr "" + +msgid "Date Format" +msgstr "" + +msgid "Text Attribute Date Format" +msgstr "" + +msgid "Best Guess" +msgstr "" + +msgid "Custom" +msgstr "" + +msgid "Optional End-Time attribute" +msgstr "" + +msgid "Existing Time Attribute" +msgstr "" + +msgid "Convert Text Attribute" +msgstr "" + +msgid "Convert Number (As Year)" +msgstr "" + +msgid "Time attribute Presentation" +msgstr "" + +msgid "List" +msgstr "" + +msgid "of all the distinct time values" +msgstr "" + +msgid "Intervals" +msgstr "" + +msgid "defined by the resolution" +msgstr "" + +msgid "Continuous Intervals" +msgstr "" + +msgid "" +"for data that is frequently updated, resolution describes the frequency of " +"updates" +msgstr "" + +msgid "Resolution of time attribute" +msgstr "" + +msgid "Enabling Time" +msgstr "" + +msgid "" +"A dataset can support one or two time attributes. If a single\n" +" attribute is used, the dataset is considered to " +"contain data that is valid at single points in time. If two\n" +" attributes are used, the second attribute represents " +"the end of a valid period hence the dataset is considered\n" +" to contain data that is valid at certain periods in " +"time." +msgstr "" + +msgid "Selecting an Attribute" +msgstr "" + +msgid "A time attribute can be" +msgstr "" + +msgid "An existing date" +msgstr "" + +msgid "Text that can be converted to a timestamp" +msgstr "" + +msgid "A number representing a year" +msgstr "" + +msgid "" +"\n" +" For text attributes, one can specify a custom format " +"(as part of the \"Advanced Options\") or use the 'best guess' approach which " +"will try to\n" +" automatically translate well-known recognized " +"patterns into valid times.\n" +" " +msgstr "" + +msgid "The 'best guess' will handle date and optional time variants of" +msgstr "" + +msgid "In terms of the formatting flags noted above, these are" +msgstr "" + +msgid "Modal Header" +msgstr "" + +msgid "Some text in the modal" +msgstr "" + +msgid " does NOT match any valid ISO-8601 Date-Time string!" +msgstr "" + +msgid "Valid ISO-8601 for" +msgstr "" + +msgid "matches a valid ISO-8601 Date-Time string!" +msgstr "" + +msgid "Not Valid ISO-8601 for" +msgstr "" + +msgid "does NOT match any valid ISO-8601 Date-Time string!" +msgstr "" + +msgid "Wrong Selection" +msgstr "" + +msgid "Please, select one Time Attribute to test!" +msgstr "" + +msgid "" +"Returning to the upload starting page in 5seconds " +msgstr "" + +msgid " Or just go " +msgstr "" + +msgid "now" +msgstr "" + +#, python-brace-format +msgid "You are attempting to {action_type} a vector dataset with a raster." +msgstr "" + +#, python-brace-format +msgid "You are attempting to {action_type} a raster dataset with a vector." +msgstr "" + +#, python-brace-format +msgid "" +"You are attempting to {action_type} a vector dataset with an unknown format." +msgstr "" + +#, python-brace-format +msgid "" +"Please ensure the name is consistent with the file you are trying to " +"{action_type}." +msgstr "" + +msgid "Local GeoNode dataset has no geometry type." +msgstr "" + +#, python-brace-format +msgid "" +"Please ensure there is at least one geometry " +"type that is consistent with the file you are " +"trying to {action_type}." +msgstr "" + +msgid "The selected Dataset does not exists in the catalog." +msgstr "" + +msgid "Please ensure that the dataset structure is consistent " +msgstr "" + +msgid "" +"Some error occurred while trying to access the uploaded schema: {str(e)}" +msgstr "" + +msgid "" +"There was an error while attempting to upload your data. Please try again, " +"or contact and administrator if the problem continues." +msgstr "" + +msgid "" +"Note: this dataset's orginal metadata was populated and preserved by " +"importing a metadata XML file. This metadata cannot be edited." +msgstr "" + +msgid "You are not permitted to delete this dataset" +msgstr "" + +msgid "You do not have permissions for this dataset." +msgstr "" + +msgid "You are not permitted to modify this dataset" +msgstr "" + +msgid "You are not permitted to modify this dataset's metadata" +msgstr "" + +msgid "You are not permitted to view this dataset" +msgstr "" + +msgid "" +"This dataset is a member of a layer group, you must remove the dataset from " +"the group before deleting." +msgstr "" + +#, python-format +msgid "couldn't generate thumbnail: %s" +msgstr "" + +msgid "Map Created" +msgstr "" + +msgid "A Map was created" +msgstr "" + +msgid "Map Updated" +msgstr "" + +msgid "A Map was updated" +msgstr "" + +msgid "Map Approved" +msgstr "" + +msgid "A Map was approved by a Manager" +msgstr "" + +msgid "Map Published" +msgstr "" + +msgid "A Map was published" +msgstr "" + +msgid "Map Deleted" +msgstr "" + +msgid "A Map was deleted" +msgstr "" + +msgid "Comment on Map" +msgstr "" + +msgid "A map was commented on" +msgstr "" + +msgid "Rating for Map" +msgstr "" + +msgid "A rating was given to a map" +msgstr "" + +msgid "Featured Map URL" +msgstr "" + +msgid "stack order" +msgstr "" + +msgid "format" +msgstr "" + +msgid "name" +msgstr "" + +msgid "store" +msgstr "" + +msgid "opacity" +msgstr "" + +msgid "styles" +msgstr "" + +msgid "transparent" +msgstr "" + +msgid "fixed" +msgstr "" + +msgid "group" +msgstr "" + +msgid "visibility" +msgstr "" + +msgid "ows URL" +msgstr "" + +msgid "layer params" +msgstr "" + +msgid "source params" +msgstr "" + +msgid "Map layers WMS GetCapabilities document" +msgstr "" + +msgid "Rate this Map" +msgstr "" + +msgid "Download Map" +msgstr "" + +msgid "Map" +msgid_plural "Maps" +msgstr[0] "" +msgstr[1] "" + +msgid "Map Layers" +msgstr "" + +msgid "This map uses the following layers:" +msgstr "" + +msgid "Documents related to this map" +msgstr "" + +msgid "List of documents related to this map:" +msgstr "" + +msgid "Change Permissions of this Map" +msgstr "" + +msgid "Copy this map" +msgstr "" + +msgid "Duplicate this map and modify it for your own purposes" +msgstr "" + +msgid "Create a New Map" +msgstr "" + +msgid "" +"\n" +" Here you can download all the layers of this map that\n" +" are hosted on this GeoNode.\n" +" " +msgstr "" + +msgid "" +"\n" +"
          Could not find downloadable layers " +"for this map. You can go back to \n" +" " +msgstr "" + +msgid "" +"\n" +" Additionally, the map contains these layers which will not be " +"downloaded\n" +" due to security restrictions:\n" +" " +msgstr "" + +msgid "" +"\n" +" Finally, the map contains these layers which will not be downloaded\n" +" because they are not available directly from this GeoNode:\n" +" " +msgstr "" + +msgid "Downloading" +msgstr "" + +msgid "Retry" +msgstr "" + +msgid "Start downloading this map" +msgstr "" + +msgid "Download Complete" +msgstr "" + +msgid "Download Failed" +msgstr "" + +msgid "Edit Map" +msgstr "" + +msgid "Explore Maps" +msgstr "" + +msgid "" +"Note: this map's orginal metadata was populated by importing a metadata XML " +"file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" + +msgid "Return to Map" +msgstr "" + +msgid "Remove Map" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(map_title)s?\n" +" " +msgstr "" + +msgid "You are not permitted to delete this map." +msgstr "" + +msgid "You do not have permissions for this map." +msgstr "" + +msgid "You must be logged in to save this map" +msgstr "" + +msgid "You are not permitted to save or edit this map." +msgstr "" + +msgid "You are not allowed to modify this map's metadata." +msgstr "" + +msgid "You are not allowed to view this map." +msgstr "" + +msgid "error saving thumbnail" +msgstr "" + +msgid "Message received" +msgstr "" + +msgid "New message received in one of your threads" +msgstr "" + +msgid "Monitoring alert" +msgstr "" + +msgid "Alert situation reported by monitoring" +msgstr "" + +msgid "GeoNode Monitoring on {base_ctx['host']} " +msgstr "" + +msgid "Enter valid email addresses." +msgstr "" + +msgid "Show list of services" +msgstr "" + +msgid "" +"Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, last sync will be used." +msgstr "" + +msgid "" +"Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, now will be used." +msgstr "" + +msgid "Force check" +msgstr "" + +msgid "Format of audit log (xml, json)" +msgstr "" + +msgid "" +"Should old data be preserved (default: no, data older than settings." +"MONITORING_DATA_TTL will be removed)" +msgstr "" + +msgid "Should stop on first error occured (default: no)" +msgstr "" + +msgid "Should process and send notifications as well (default: no)" +msgstr "" + +msgid "Collect data from this service only" +msgstr "" + +msgid "Show list of metrics" +msgstr "" + +msgid "Show list of labels for metric" +msgstr "" + +msgid "Show list of resources for metric" +msgstr "" + +msgid "Data aggregation interval in seconds (default: 60)" +msgstr "" + +msgid "Metric name" +msgstr "" + +msgid "Show data for specific resource in resource_type=resource_name format" +msgstr "" + +msgid "Show data for specific resource" +msgstr "" + +msgid "Show data for specific label" +msgstr "" + +#, python-brace-format +msgid "Write result to file, default GEOIP_PATH: {settings.GEOIP_PATH}" +msgstr "" + +msgid "" +"Fetch database from specific url. If nothing provided, default {} will be " +"used" +msgstr "" + +msgid "Overwrite file if exists" +msgstr "" + +msgid "GeoNode" +msgstr "" + +msgid "GeoServer" +msgstr "" + +msgid "Host (GeoServer)" +msgstr "" + +msgid "Host (GeoNode)" +msgstr "" + +msgid "No resource" +msgstr "" + +msgid "Resource base" +msgstr "" + +msgid "Style" +msgstr "" + +msgid "Admin" +msgstr "" + +msgid "Other" +msgstr "" + +msgid "Rate" +msgstr "" + +msgid "Count" +msgstr "" + +msgid "Value" +msgstr "" + +msgid "Value numeric" +msgstr "" + +msgid "Bytes" +msgstr "" + +msgid "Kilobytes" +msgstr "" + +msgid "Megabytes" +msgstr "" + +msgid "Gigabytes" +msgstr "" + +msgid "Bytes per second" +msgstr "" + +msgid "Kilobytes per second" +msgstr "" + +msgid "Megabytes per second" +msgstr "" + +msgid "Gigabytes per second" +msgstr "" + +msgid "Seconds" +msgstr "" + +msgid "Percentage" +msgstr "" + +msgid "Not OWS" +msgstr "" + +msgid "Any OWS" +msgstr "" + +msgid "Change" +msgstr "" + +msgid "Change Metadata" +msgstr "" + +msgid "View Metadata" +msgstr "" + +msgid "Publish" +msgstr "" + +msgid "Geoserver event" +msgstr "" + +msgid "List of resources affected" +msgstr "" + +msgid "Response processing time in ms" +msgstr "" + +msgid "1 minute" +msgstr "" + +msgid "5 minutes" +msgstr "" + +msgid "10 minutes" +msgstr "" + +msgid "30 minutes" +msgstr "" + +msgid "1 hour" +msgstr "" + +msgid "Error" +msgstr "" + +msgid "Fatal" +msgstr "" + +msgid "Expected min/max values for user configuration" +msgstr "" + +msgid "Marker of last delivery" +msgstr "" + +msgid "Minimum time between subsequent notifications" +msgstr "" + +msgid "How severe would be error from this notification" +msgstr "" + +msgid "Is it active" +msgstr "" + +#, python-brace-format +msgid "{self.severity}: {self.name}" +msgstr "" + +msgid "Value must be above" +msgstr "" + +msgid "Value must be below" +msgstr "" + +msgid "Last update must not be older than" +msgstr "" + +msgid "Max timeout for given metric before error should be raised" +msgstr "" + +msgid "Monitoring & Analytics" +msgstr "" + +msgid "Only superusers can monitor performance" +msgstr "" + +msgid "Hi," +msgstr "" + +msgid "Monitoring on" +msgstr "" + +msgid "has detected following problems :" +msgstr "" + +msgid "spotted" +msgstr "" + +msgid "ago" +msgstr "" + +msgid "emerged" +msgstr "" + +msgid "User following you" +msgstr "" + +msgid "Another user has started following you" +msgstr "" + +msgid "User requested access" +msgstr "" + +msgid "A new user has requested access to the site" +msgstr "" + +msgid "Account activated" +msgstr "" + +msgid "This account is now active and can log in the site" +msgstr "" + +msgid "Personal info" +msgstr "" + +msgid "Important dates" +msgstr "" + +msgid "Extended profile" +msgstr "" + +msgid "Password changed successfully." +msgstr "" + +#, python-format +msgid "Change password: %s" +msgstr "" + +msgid "party who authored the resource" +msgstr "" + +msgid "" +"party who has processed the data in a manner such that the resource has been " +"modified" +msgstr "" + +msgid "party who published the resource" +msgstr "" + +msgid "" +"party that accepts accountability and responsibility for the data and " +"ensures appropriate care and maintenance of the resource" +msgstr "" + +msgid "" +"party who can be contacted for acquiring knowledge about or acquisition of " +"the resource" +msgstr "" + +msgid "party who distributes the resource" +msgstr "" + +msgid "party who uses the resource" +msgstr "" + +msgid "party that supplies the resource" +msgstr "" + +msgid "party who created the resource" +msgstr "" + +msgid "party that owns the resource" +msgstr "" + +msgid "key party responsible for gathering information and conducting research" +msgstr "" + +msgid "Email Address" +msgstr "" + +msgid "Organization Name" +msgstr "" + +msgid "name of the responsible organization" +msgstr "" + +msgid "Profile" +msgstr "" + +msgid "introduce yourself" +msgstr "" + +msgid "Position Name" +msgstr "" + +msgid "role or position of the responsible person" +msgstr "" + +msgid "Voice" +msgstr "" + +msgid "" +"telephone number by which individuals can speak to the responsible " +"organization or individual" +msgstr "" + +msgid "Facsimile" +msgstr "" + +msgid "" +"telephone number of a facsimile machine for the responsible organization or " +"individual" +msgstr "" + +msgid "Delivery Point" +msgstr "" + +msgid "" +"physical and email address at which the organization or individual may be " +"contacted" +msgstr "" + +msgid "City" +msgstr "" + +msgid "city of the location" +msgstr "" + +msgid "Administrative Area" +msgstr "" + +msgid "state, province of the location" +msgstr "" + +msgid "Postal Code" +msgstr "" + +msgid "ZIP or other postal code" +msgstr "" + +msgid "Country" +msgstr "" + +msgid "country of the physical address" +msgstr "" + +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated" +msgstr "" + +msgid "Timezone" +msgstr "" + +msgid "No Group" +msgstr "" + +msgid "All contents" +msgstr "" + +msgid "No Organization Info" +msgstr "" + +msgid "Forgot Username" +msgstr "" + +msgid "" +"Enter your email address and click the submit button.
          Your username " +"will be sent to you." +msgstr "" + +msgid "Create Profile" +msgstr "" + +msgid "Profile of " +msgstr "" + +msgid "Not provided." +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Organization" +msgstr "" + +msgid "Not provided" +msgstr "" + +msgid "Fax" +msgstr "" + +msgid "User layers WMS GetCapabilities document" +msgstr "" + +msgid "Message User" +msgstr "" + +msgid "Edit profile" +msgstr "" + +msgid "Connected social accounts" +msgstr "" + +msgid "Associated e-mails" +msgstr "" + +msgid "Set/Change password" +msgstr "" + +msgid "Upload new layers" +msgstr "" + +msgid "Create a new layer" +msgstr "" + +msgid "Upload new document" +msgstr "" + +msgid "Create a new map" +msgstr "" + +msgid "My Activities" +msgstr "" + +msgid "Notifications" +msgstr "" + +msgid "Announcements" +msgstr "" + +msgid "Invite Users" +msgstr "" + +msgid "User Activities" +msgstr "" + +msgid "Resources" +msgstr "" + +msgid "Edit Your Profile" +msgstr "" + +msgid "Edit Profile for" +msgstr "" + +msgid "Change your avatar" +msgstr "" + +msgid "Update profile" +msgstr "" + +msgid "Explore People" +msgstr "" + +#, python-brace-format +msgid "Your username for {site.name}" +msgstr "" + +msgid "Your username has been emailed to you." +msgstr "" + +msgid "No user could be found with that email address." +msgstr "" + +msgid "You are not permitted to save or edit this resource." +msgstr "" + +msgid "You are not authorized to download this resource." +msgstr "" + +msgid "" +"No files have been found for this resource. Please, contact a system " +"administrator." +msgstr "" + +msgid "No files found." +msgstr "" + +msgid "Not Authorized" +msgstr "" + +msgid "GeoNode Resource Processing Library" +msgstr "" + +msgid "Disabling this Task will make the Processing Workflow to skip it." +msgstr "" + +msgid "Processing Workflow Tasks" +msgstr "" + +msgid "Session is Expired. Please login again!" +msgstr "" + +msgid "Permissions successfully updated!" +msgstr "" + +msgid "Error updating permissions :(" +msgstr "" + +msgid "You are not allowed to change permissions for this resource" +msgstr "" + +msgid "Security Rules Cache Refreshed!" +msgstr "" + +msgid "You cannot modify this resource!" +msgstr "" + +msgid "Attributes/Stats Refreshed Successfully!" +msgstr "" + +msgid "GeoWebCache Tiled Layer Emptied!" +msgstr "" + +msgid "Wrong permissions specification" +msgstr "" + +msgid "error delivering notification" +msgstr "" + +msgid "Auto-detect" +msgstr "" + +msgid "Paired WMS/WFS/WCS" +msgstr "" + +msgid "Web Map Service" +msgstr "" + +msgid "Catalogue Service" +msgstr "" + +msgid "ArcGIS REST MapServer" +msgstr "" + +msgid "ArcGIS REST ImageServer" +msgstr "" + +msgid "OpenGeoPortal" +msgstr "" + +msgid "Harvard Geospatial Library" +msgstr "" + +msgid "GeoNode (Web Map Service)" +msgstr "" + +msgid "GeoNode (Catalogue Service)" +msgstr "" + +msgid "Service URL" +msgstr "" + +msgid "Service Type" +msgstr "" + +#, python-format +msgid "Service %(url)s is already registered" +msgstr "" + +#, python-format +msgid "Could not connect to the service at %(url)s" +msgstr "" + +#, python-format +msgid "Could not find importable resources for the service at %(url)s" +msgstr "" + +#, python-format +msgid "Found service of type %(found_type)s instead of %(service_type)s" +msgstr "" + +msgid "Local" +msgstr "" + +msgid "Cascaded" +msgstr "" + +msgid "Harvested" +msgstr "" + +msgid "Indexed" +msgstr "" + +msgid "Live" +msgstr "" + +msgid "Resource is queued" +msgstr "" + +msgid "Remote service unavailable" +msgstr "" + +msgid "Service unavailable" +msgstr "" + +msgid "Could not contact" +msgstr "" + +msgid "Contact" +msgstr "" + +msgid "Service Resources" +msgstr "" + +msgid "No resources have been imported yet." +msgstr "" + +msgid "Retry job" +msgstr "" + +msgid "previous" +msgstr "" + +msgid "next" +msgstr "" + +msgid "Connecting to service..." +msgstr "" + +msgid "Edit Service Metadata" +msgstr "" + +msgid "Import Service Resources" +msgstr "" + +msgid "Remove Service" +msgstr "" + +msgid "Edit Service" +msgstr "" + +msgid "Services" +msgstr "" + +msgid "Add Remote Service" +msgstr "" + +msgid "Remote Services" +msgstr "" + +msgid "No services registered" +msgstr "" + +msgid "Register Service" +msgstr "" + +msgid "Register New Service" +msgstr "" + +msgid "Service has been created:" +msgstr "" + +msgid "The following layers will be imported" +msgstr "" + +msgid "Remove Remote Service" +msgstr "" + +msgid "Are you sure you want to remove" +msgstr "" + +msgid "Deleting service and its associated resources..." +msgstr "" + +msgid "Import resources" +msgstr "" + +msgid "no" +msgstr "" + +msgid "resources can be imported" +msgstr "" + +msgid "- These will be cascaded through your local geoserver instance" +msgstr "" + +msgid "Id" +msgstr "" + +msgid "Clear Filter" +msgstr "" + +msgid "Back to service details" +msgstr "" + +msgid "Import Resources" +msgstr "" + +msgid "Previous" +msgstr "" + +msgid "All resources have already been imported" +msgstr "" + +msgid "Harvesting resources..." +msgstr "" + +msgid "Re-scan Service for new Resources" +msgstr "" + +msgid "Service registered successfully" +msgstr "" + +msgid "The selected resources are being imported" +msgstr "" + +msgid "The selected resources have been imported" +msgstr "" + +msgid "Resource is already being processed" +msgstr "" + +#, python-brace-format +msgid "Resource {resource_id} is being processed" +msgstr "" + +msgid "Service rescanned successfully" +msgstr "" + +msgid "You dont have enougth rigths to see the resource detail" +msgstr "" + +msgid "You are not permitted to change this service." +msgstr "" + +msgid "You are not permitted to remove this service." +msgstr "" + +#, python-brace-format +msgid "Service {service.name} has been deleted" +msgstr "" + +msgid "created" +msgstr "" + +msgid "deleted" +msgstr "" + +msgid "updated" +msgstr "" + +msgid "added a comment" +msgstr "" + +msgid "updated a comment" +msgstr "" + +msgid "uploaded" +msgstr "" + +msgid "Recent activity" +msgstr "" + +msgid "to" +msgstr "" + +msgid "on" +msgstr "" + +msgid "Not Permitted" +msgstr "" + +msgid "" +"\n" +" You are not allowed to perform this operation.\n" +" " +msgstr "" + +msgid "Please verify that you are logged in as the correct user." +msgstr "" + +msgid "Please login or register and retry." +msgstr "" + +msgid "Page Not Found" +msgstr "" + +msgid "" +"\n" +" The page you requested does not exist. Perhaps you are using an " +"outdated bookmark?\n" +" " +msgstr "" + +msgid "Toggle navigation" +msgstr "" + +msgid "Datasets" +msgstr "" + +msgid "People" +msgstr "" + +msgid "Search" +msgstr "" + +msgid "Register" +msgstr "" + +msgid "Sign in" +msgstr "" + +msgid "You are using an outdated browser that is not supported by GeoNode." +msgstr "" + +msgid "" +"Please use a modern browser like Mozilla Firefox, Google " +"Chrome or Safari." +msgstr "" + +msgid "There was a problem loading this page" +msgstr "" + +msgid "" +"\n" +" Please contact your GeoNode administrator (they may have " +"received an email automatically if they configured it properly).\n" +" If you are the site administrator, enable debug mode to see the " +"actual error and fix it or file an issue in GeoNode's issue tracker\n" +" " +msgstr "" + +msgid "Create a new dataset" +msgstr "" + +msgid "Create Map" +msgstr "" + +msgid "Add User" +msgstr "" + +msgid "Powered by" +msgstr "" + +msgid "version" +msgstr "" + +msgid "Developers" +msgstr "" + +msgid "Username" +msgstr "" + +msgid "Password" +msgstr "" + +msgid "Remember Me" +msgstr "" + +msgid "Menu" +msgstr "" + +msgid "Recent Activity" +msgstr "" + +msgid "Inbox" +msgstr "" + +msgid "Help" +msgstr "" + +msgid "Log out" +msgstr "" + +msgid "Info" +msgstr "" + +msgid "Granules" +msgstr "" + +msgid "Share" +msgstr "" + +msgid "Ratings" +msgstr "" + +msgid "Exif" +msgstr "" + +msgid "Set permissions on selected resources" +msgstr "" + +msgid "Apply Changes" +msgstr "" + +msgid "total" +msgstr "" + +msgid "Add Comment" +msgstr "" + +msgid "Log in to add a comment" +msgstr "" + +msgid "By" +msgstr "" + +msgid "Submit Comment" +msgstr "" + +msgid "Who can view it?" +msgstr "" + +msgid "Anyone" +msgstr "" + +msgid "The following users:" +msgstr "" + +msgid "The following groups:" +msgstr "" + +msgid "Who can download it?" +msgstr "" + +msgid "Who can change metadata for it?" +msgstr "" + +msgid "Who can edit data for this dataset?" +msgstr "" + +msgid "Who can edit styles for this dataset?" +msgstr "" + +msgid "" +"Who can manage it? (update, delete, change permissions, publish/unpublish it)" +msgstr "" + +msgid "Set permissions for this resource" +msgstr "" + +msgid "People and Groups" +msgstr "" + +msgid "Geo Limits" +msgstr "" + +msgid "Users" +msgstr "" + +msgid "Show Geo Limits" +msgstr "" + +msgid "Upload Geo Limits" +msgstr "" + +msgid "Save Geo Limits" +msgstr "" + +msgid "Delete Geo Limits" +msgstr "" + +msgid "Message box" +msgstr "" + +msgid "OK" +msgstr "" + +msgid "Mandatory files : SHP , DBF" +msgstr "" + +msgid "" +"Upload a ZIP file containing an ESRI Shapefile. If the ZIP provides also a ." +"prj file, you don't have to specify the EPSG SRID" +msgstr "" + +msgid "Choose" +msgstr "" + +msgid "Choose a file..." +msgstr "" + +msgid "Reset" +msgstr "" + +msgid "Only the SRID number without the 'EPSG:' prefix!" +msgstr "" + +msgid "Default : 4326" +msgstr "" + +msgid "Set your desired encoding UTF-8, Big5, Big5-HKSCS ..." +msgstr "" + +msgid "Load SHP-ZIP" +msgstr "" + +msgid "" +"

          This will remove the Geo Limits currently drawn on the map.

          In " +"order to store the Geo Limits you will need to save them " +"anyway.

          Do you want to proceed?

          " +msgstr "" + +msgid "" +"

          This will override the current stored Geo Limits on the DB.

          To " +"apply them you will need to click on Apply Changes button " +"anyway.

          WARNING: This operation cannot be reverted!

          Do you want to proceed?

          " +msgstr "" + +msgid "Save Geo Limit" +msgstr "" + +msgid "" +"

          Geometry successfully saved!

          To apply them you " +"will need to click on Apply Changes button.

          " +msgstr "" + +msgid "

          Error while trying to save the Geometry!

          " +msgstr "" + +msgid "Load Geo Limits" +msgstr "" + +msgid "Load Geo Limit" +msgstr "" + +msgid "

          No Geometry found!

          " +msgstr "" + +msgid "" +"

          This will permanently remove the Geo Limits from the DB." +"

          To apply them you will need to click on Apply Changes button.

          Do you want to proceed?

          " +msgstr "" + +msgid "Delete Geo Limit" +msgstr "" + +msgid "

          Geo Limitsy successfully deleted!

          " +msgstr "" + +msgid "" +"

          Error occurred while trying to delete Geo Limits:

          " +msgstr "" + +msgid "Choose users..." +msgstr "" + +msgid "Choose groups..." +msgstr "" + +msgid "Placeholder for status-message" +msgstr "" + +msgid "Placeholder for status-message-body" +msgstr "" + +msgid "About GeoNode" +msgstr "" + +msgid "" +"GeoNode is a geospatial content management system, a platform for the " +"management and publication of geospatial data. It brings together mature and " +"stable open-source software projects under a consistent and easy-to-use " +"interface allowing non-specialized users to share data and create " +"interactive maps." +msgstr "" + +msgid "" +"Data management tools built into GeoNode allow for integrated creation of " +"data, metadata, and map visualizations. Each dataset in the system can be " +"shared publicly or restricted to allow access to only specific users. Social " +"features like user profiles and commenting and rating systems allow for the " +"development of communities around each platform to facilitate the use, " +"management, and quality control of the data the GeoNode instance contains." +msgstr "" + +msgid "" +"It is also designed to be a flexible platform that software developers can " +"extend, modify or integrate against to meet requirements in their own " +"applications." +msgstr "" + +msgid "Account Inactive" +msgstr "" + +msgid "This account is inactive." +msgstr "" + +msgid "Account Pending Approval" +msgstr "" + +#, python-format +msgid "" +"We have sent the administrators a notice to approve your account associated " +"with %(email)s. If the account is approved, you will receive a " +"confirmation notice." +msgstr "" + +msgid "Account" +msgstr "" + +msgid "E-mail Addresses" +msgstr "" + +msgid "The following e-mail addresses are associated with your account:" +msgstr "" + +msgid "Verified" +msgstr "" + +msgid "Unverified" +msgstr "" + +msgid "Primary" +msgstr "" + +msgid "Make Primary" +msgstr "" + +msgid "Re-send Verification" +msgstr "" + +msgid "Warning:" +msgstr "" + +msgid "" +"You currently do not have any e-mail address set up. You should really add " +"an e-mail address so you can receive notifications, reset your password, etc." +msgstr "" + +msgid "Add E-mail Address" +msgstr "" + +msgid "Add E-mail" +msgstr "" + +msgid "Do you really want to remove the selected e-mail address?" +msgstr "" + +#, python-format +msgid "" +"Hello from %(site_name)s!\n" +"\n" +"You're receiving this e-mail because user %(user_display)s has given yours " +"as an e-mail address to connect their account.\n" +"\n" +"To confirm this is correct, go to %(activate_url)s\n" +msgstr "" + +#, python-format +msgid "" +"Thank you from %(site_name)s!\n" +"%(site_domain)s" +msgstr "" + +#, python-format +msgid "Please confirm email address for %(site_name)s" +msgstr "" + +#, python-format +msgid "You have been invited to sign up at %(site_name)s." +msgstr "" + +#, python-format +msgid "Create an account on %(site_name)s" +msgstr "" + +msgid "" +"This is the email notification to confirm your password has been changed on" +msgstr "" + +msgid "Change password email notification" +msgstr "" + +#, python-format +msgid "" +"Hello from %(site_name)s!\n" +"\n" +"You're receiving this e-mail because you or someone else has requested a " +"password for your user account.\n" +"It can be safely ignored if you did not request a password reset. Click the " +"link below to reset your password." +msgstr "" + +#, python-format +msgid "In case you forgot, your username is %(username)s." +msgstr "" + +#, python-format +msgid "" +"Thank you for using %(site_name)s!\n" +"%(site_domain)s" +msgstr "" + +#, python-format +msgid "[%(site_name)s] Password reset" +msgstr "" + +msgid "Confirm E-mail Address" +msgstr "" + +#, python-format +msgid "" +"Please confirm that %(email)s is an e-mail " +"address for user %(user_display)s." +msgstr "" + +msgid "Confirm" +msgstr "" + +#, python-format +msgid "" +"This e-mail confirmation link expired or is invalid. Please issue a new e-mail confirmation request." +msgstr "" + +msgid "Log in" +msgstr "" + +msgid "Log in to an existing account" +msgstr "" + +#, python-format +msgid "" +"Please sign in with one\n" +" of your existing third party accounts. Or, sign up\n" +" for a %(site_name)s account and sign in below:" +msgstr "" + +msgid "or" +msgstr "" + +#, python-format +msgid "" +"If you have not created an account yet, then please\n" +" sign up first." +msgstr "" + +msgid "Forgot Password?" +msgstr "" + +msgid "Sign In" +msgstr "" + +msgid "Are you sure you want to log out?" +msgstr "" + +msgid "Password change" +msgstr "" + +msgid "Password Change" +msgstr "" + +msgid "Change your password here" +msgstr "" + +msgid "Change my password" +msgstr "" + +msgid "Password reset" +msgstr "" + +msgid "Password Reset" +msgstr "" + +msgid "Forgotten your password?" +msgstr "" + +msgid "" +"Enter your email address below, and we'll send you an email allowing you to " +"reset it." +msgstr "" + +msgid "Reset my password" +msgstr "" + +#, python-format +msgid "" +"If you have any trouble resetting your password, contact us at %(THEME_ACCOUNT_CONTACT_EMAIL)s." +msgstr "" + +msgid "" +"We have sent you an e-mail. Please contact us if you do not receive it " +"within a few minutes." +msgstr "" + +msgid "Change Password" +msgstr "" + +msgid "Bad Token" +msgstr "" + +#, python-format +msgid "" +"The password reset link was invalid, possibly because it has already been " +"used. Please request a new password reset." +msgstr "" + +msgid "change password" +msgstr "" + +msgid "Your password is now changed." +msgstr "" + +msgid "Set Password" +msgstr "" + +msgid "Password Set" +msgstr "" + +msgid "Set your password here" +msgstr "" + +msgid "Set password" +msgstr "" + +msgid "Sign up" +msgstr "" + +msgid "" +"Sign up with one\n" +" of your existing third party accounts" +msgstr "" + +msgid "Create a new local account" +msgstr "" + +msgid "Sign Up Closed" +msgstr "" + +msgid "We are sorry, but the sign up is currently closed." +msgstr "" + +msgid "Note" +msgstr "" + +#, python-format +msgid "you are already logged in as %(user_display)s." +msgstr "" + +msgid "Verify your email address" +msgstr "" + +msgid "" +"We have sent an email to you for verification. Follow the\n" +" link provided to finalize the signup process. Please contact us if you " +"do\n" +" not receive it within a few minutes" +msgstr "" + +msgid "Verify Your E-mail Address" +msgstr "" + +msgid "" +"This part of the site requires us to verify that\n" +"you are who you claim to be. For this purpose, we require that you\n" +"verify ownership of your e-mail address. " +msgstr "" + +msgid "" +"We have sent an e-mail to you for\n" +"verification. Please click on the link inside this e-mail. Please\n" +"contact us if you do not receive it within a few minutes." +msgstr "" + +#, python-format +msgid "" +"Note: you can still change your e-" +"mail address." +msgstr "" + +msgid "My Activity feed" +msgstr "" + +msgid "Activity Feed" +msgstr "" + +msgid "Are you sure you want to run the selected" +msgstr "" + +msgid "Yes, I'm sure" +msgstr "" + +msgid "GeoNode site admin" +msgstr "" + +msgid "GeoNode administration" +msgstr "" + +msgid "Delete Announcement?" +msgstr "" + +msgid "Are you sure you want to delete this announcement?" +msgstr "" + +msgid "Edit Announcement" +msgstr "" + +msgid "Create Announcement" +msgstr "" + +msgid "New Announcement" +msgstr "" + +msgid "Level" +msgstr "" + +msgid "Announcement" +msgstr "" + +msgid "Published From" +msgstr "" + +#, python-format +msgid "" +"\n" +" Published from %(publish_start)s to %(publish_end)s.\n" +" " +msgstr "" + +#, python-format +msgid "Announcements : %(announcement)s" +msgstr "" + +msgid "Back to edit your profile information" +msgstr "" + +msgid "Your current avatar: " +msgstr "" + +msgid "You haven't uploaded an avatar yet. Please upload one now." +msgstr "" + +msgid "Upload New Image" +msgstr "" + +msgid "Choose new Default" +msgstr "" + +msgid "Delete Your Avatar" +msgstr "" + +msgid "Please select the avatars that you would like to delete." +msgstr "" + +#, python-format +msgid "" +"You have no avatars to delete. Please upload one now." +msgstr "" + +msgid "Delete These" +msgstr "" + +msgid "GeoNode Search" +msgstr "" + +msgid "Contact Us" +msgstr "" + +msgid "Tel" +msgstr "" + +msgid "Uploading..." +msgstr "" + +msgid "Upload in progress..." +msgstr "" + +msgid "Processing..." +msgstr "" + +msgid "Updating Thumbnail..." +msgstr "" + +msgid "Message. Do you want to proceed?" +msgstr "" + +msgid "Processing Resource..." +msgstr "" + +msgid "Information for Developers" +msgstr "" + +msgid "Useful information for developers interested in GeoNode." +msgstr "" + +#, python-format +msgid "" +"\n" +"

          GeoNode is an open service built " +"on open source software. We encourage you to build new applications using " +"the components and resources it provides. This page is a starting point for " +"developers interesting in taking full advantage of GeoNode. It also includes " +"links to the project's source code so anyone can build and customize their " +"own GeoNode.

          \n" +"\n" +"

          GeoNode Software

          \n" +"\n" +"

          All the code that runs GeoNode is open source. The code is available " +"at http://github.com/GeoNode/" +"geonode/. The issue tracker for the project is at http://github.com/GeoNode/geonode/" +"issues.

          \n" +"\n" +"

          GeoNode is built using several open source projects, each with its " +"own community. If you are interested in contributing new features to the " +"GeoNode, we encourage you to do so by contributing to one of the projects on " +"which it is built:

          \n" +"
            \n" +"
          • GeoServer - Standards based " +"server for geospatial information
            • \n" +"
          • GeoWebCache - Cache engine " +"for WMS Tiles
          • OpenLayers - " +"Pure JavaScript library powering the maps of GeoExt
            • \n" +"
          • pycsw - CSW, OpenSearch and OAI-" +"PMH metadata catalogue server
            • \n" +"
          \n" +"\n" +"

          What are OGC Services?

          \n" +"

          The data in this application is served using open standards endorsed " +"by ISO and the Open Geospatial " +"Consortium; in particular, WMS (Web Map Service) is used for accessing " +"maps, WFS (Web Feature Service) is used for accessing vector data, and WCS " +"(Web Coverage Service) is used for accessing raster data. WMC (Web Map " +"Context Documents) is used for sharing maps. You can use these services in " +"your own applications using libraries such as OpenLayers, GeoTools, and OGR " +"(all of which are open-source software and available at zero cost). " +"Additionally, CSW (Catalog Service for the Web) supports access to " +"collections of descriptive information (metadata) about data and services.\n" +"\n" +"

          What is GeoWebCache?

          \n" +"

          GeoWebCache provides mapping tiles that are compatible with a number " +"of mapping engines, including Google Maps, Bing Maps and OpenLayers. All the " +"data hosted by GeoNode is also available through GeoWebCache. GeoWebCache " +"improves on WMS by caching data and providing more responsive maps.

          \n" +"\n" +"

          CSW Example Code

          \n" +"

          To interact with GeoNode's CSW you can use any CSW client (QGIS " +"MetaSearch, GRASS, etc.). The following example illustrates a simple " +"invocation using the OWSLib Python package:

          \n" +"

          from owslib.csw import CatalogueServiceWeb

          \n" +"

          from owslib.fes import PropertyIsLike

          \n" +"

          csw = CatalogueServiceWeb('%(CATALOGUE_BASE_URL)s')

          \n" +"

          anytext = PropertyIsLike('csw:AnyText', 'birds')')

          \n" +"

          csw.getrecords2(constraints=[anytext])

          \n" +"

          print csw.results

          \n" +"

          print csw.records

          \n" +"\n" +"

          OpenLayers Example Code

          \n" +"\n" +"

          To include a GeoNode map layer in an OpenLayers map, first find the " +"name for that layer. This is found in the layer's name field " +"(not title) of the layer list. For this example, we will use " +"the Nicaraguan political boundaries background layer, whose name is " +"risk:nicaragua_admin. Then, create an instance of OpenLayers." +"Layer.WMS:

          \n" +"

          var geonodeLayer = new OpenLayers.Layer.WMS(\"GeoNode Risk Data" +"\", \"http://demo.geonode.org/geoserver/wms\",{ layers: \"risk:" +"nicaragua_admin\" });

          \n" +"\n" +"

          Google Maps Example Code

          \n" +"

          To include a GeoNode map layer in a Google Map, include the dataset " +"name in the URL template.

          \n" +"

          var tilelayer = new GTileLayer(null, null, null, " +"{tileUrlTemplate: 'http://demo.geonode.org/geoserver/gwc/service/gmaps?" +"layers=risk:nicaragua_admin&zoom={Z}&x={X}&y={Y}', isPng:true, " +"opacity:0.5 } );

          \n" +"\n" +"

          Shapefile/GeoJSON/GML Output

          \n" +"

          To get data from the GeoNode web services use the WFS protocol. For " +"example, to get the full Nicaraguan admin boundaries use:

          \n" +"

          http://demo.geonode.org/geoserver/wfs?request=GetFeature&" +"typeName=risk:nicaragua_admin&outputformat=SHAPE-ZIP

          \n" +"

          Changing output format to json, GML2, " +"GML3, or csv will get data in those formats. The " +"WFS protocol also can handle more precise queries, specifying a bounding box " +"or various spatial and non-spatial filters based on the attributes of the " +"data.

          \n" +"\n" +"

          GeoTools Example Code

          \n" +"

          Create a DataStore and extract a FeatureType from it, then run a " +"Query. It is all documented on the wiki at http://geotools.org/." +"

          \n" +" " +msgstr "" + +msgid "GeoNode's Web Services" +msgstr "" + +msgid "GeoNode's Web Services are available from the following URLs:" +msgstr "" + +msgid "Dynamic tiles via WMS:" +msgstr "" + +msgid "Vector data via WFS:" +msgstr "" + +msgid "Raster data via WCS:" +msgstr "" + +msgid "Metadata search via CSW:" +msgstr "" + +msgid "Metadata search via OpenSearch:" +msgstr "" + +msgid "Metadata search via OAI-PMH:" +msgstr "" + +msgid "Cached tiles via WMTS:" +msgstr "" + +msgid "User Guide" +msgstr "" + +msgid "GeoNode Help" +msgstr "" + +msgid "" +"This page provides helpful information about how to use the GeoNode. You can " +"use the sidebar links to navigate to what you want to know." +msgstr "" + +msgid "" +"\n" +"

          The GeoNode provides access to data sets and a map editing " +"application allows users to browse existing maps and contribute their own.\n" +"\n" +"

          Browsing Layers

          \n" +"

          The Layers tab allows you to browse data " +"uploaded to this GeoNode.

          \n" +"

          All data can be downloaded in a variety of formats, for use in other " +"applications.

          \n" +"\n" +"

          Developer Access

          \n" +"

          The Developer page is the place for " +"developers to get started building applications against the GeoNode. It " +"includes instructions on using the web services, links to the source code of " +"the GeoNode, and information about the open source projects used to create " +"it.

          \n" +"\n" +"

          Browsing Maps

          \n" +"

          The GeoNode allows users to create and share maps with one another.\n" +"

          The Maps tab is a gateway to map exploration on " +"GeoNode. From here you can search for a map or " +"create a map, which will open the Map " +"Composer.

          \n" +"
          Google Earth Mode
          \n" +"

          Any map viewed in the interactive map editor can be seen in 3D mode " +"with the Google Earth plugin. To switch to 3D mode select the Google Earth " +"globe logo, the rightmost button on top toolbar. If you do not have the " +"Google Earth plugin installed you will be prompted to install it.

          \n" +"\n" +"

          Creating a Map

          \n" +"

          To create a new map go to the Contributed Maps " +"tab and click the create your own map link.

          \n" +"

          This will take you to the Map Composer with " +"a base layer loaded.

          \n" +"

          To add data layers from the GeoNode click on the green plus button " +"located below the layers tab on the left hand side of the screen. This will " +"open a dialog listing all the layers available on the GeoNode.

          \n" +"

          To add layers to your map select them and hit the Add Layers button. When finished you may hit Done to close the " +"dialog and go back to the map.

          \n" +"\n" +"
          Reordering and Removing Layers
          \n" +"

          Change the display order of the layers listed in the data tab by " +"simply dragging and dropping their names. The order in the map will be " +"updated to reflect that. To turn a layer's visibility off simply uncheck it, " +"and to remove it entirely select it and hit the red minus button.

          \n" +"\n" +"
          Saving your map
          \n" +"

          Once a suitable set of layers and zoom level has been found it's time " +"to save it so others can see it. Click the Save button--the left most icon " +"on the top toolbar, an image of a map with a disk--on the top menu and fill " +"out the title and abstract of the map.

          \n" +"\n" +"

          Publishing a Map

          \n" +"

          Any map from the GeoNode can be embedded for use in another site or " +"blog. To publish a map:

          \n" +"
            \n" +"
          1. Select the from the list of maps on the community or map page and " +"then hit the 'Publish map' button.
            2. \n" +"
          2. Choose your desired height and width for the widget in the wizard." +"
            3. \n" +"
          3. Copy the HTML snippet provided in the wizard to any HTML page or " +"iFrame-supporting blog post.
            4. \n" +"
          \n" +"

          This will put an interactive widget showing you map in your web page " +"or blog post.

          \n" +"

          Note that the Map Composer also has a " +"button to publish the map. Just be sure to save the map before publishing if " +"there are changes that you want others to see. It publishes the last saved " +"version, not the last viewed version.

          \n" +"\n" +"

          Remixing a Map

          \n" +"

          Any map available can serve as a starting point for a new map.

          \n" +"
            \n" +"
          1. Open the map in the Map Composer.
            2. \n" +"
          2. Add and remove layers as you like.
            3. \n" +"
          3. Pan and zoom to highlight the area of interest.
            4. \n" +"
          4. IMPORTANT: Update the title, abstract, contact " +"info and tags to reflect the new map.
            5. \n" +"
          5. Save the map.
            6. \n" +"
          \n" +"

          You will be able to see your new map when you search for it from the " +"Maps tab.

          \n" +" " +msgstr "" + +msgid "Sections" +msgstr "" + +msgid "Browsing Layers" +msgstr "" + +msgid "Developer Access" +msgstr "" + +msgid "Browsing Maps" +msgstr "" + +msgid "Creating a Map" +msgstr "" + +msgid "Exporting a Map" +msgstr "" + +msgid "Remixing a Map" +msgstr "" + +msgid "Welcome!" +msgstr "" + +msgid "Get Started »" +msgstr "" + +msgid "Search for Data." +msgstr "" + +msgid "Advanced Search" +msgstr "" + +msgid "" +"Click to search for geospatial data published by other users, organizations " +"and public sources. Download data in standard formats." +msgstr "" + +msgid "Add datasets" +msgstr "" + +msgid "Explore datasetss" +msgstr "" + +msgid "" +"Data is available for browsing, aggregating and styling to generate maps " +"which can be saved, downloaded, shared publicly or restricted to specify " +"users only." +msgstr "" + +msgid "Create maps" +msgstr "" + +msgid "Explore maps" +msgstr "" + +msgid "" +"As for the layers and maps GeoNode allows to publish tabular and text data, " +"manage their metadata and associated documents." +msgstr "" + +msgid "Add documents" +msgstr "" + +msgid "Explore documents" +msgstr "" + +msgid "" +"Geonode allows registered users to easily upload geospatial data and various " +"documents in several formats." +msgstr "" + +msgid "See users" +msgstr "" + +msgid "Featured Datasets" +msgstr "" + +msgid "Accept" +msgstr "" + +msgid "Reject" +msgstr "" + +msgid "Leave" +msgstr "" + +msgid "Dear Sir/Madam," +msgstr "" + +#, python-format +msgid "is inviting you to join (%(site_name)s)." +msgstr "" + +#, python-format +msgid "" +"To do so, please register at %(site_name)s " +"Registration." +msgstr "" + +msgid "" +"Once you receive the confirmation that your account is activated, you can " +"notify" +msgstr "" + +msgid "that you wish to join her/his group(s) through" +msgstr "" + +msgid "this link" +msgstr "" + +#, python-format +msgid "" +"%(inviter_name)s is a member of the following group(s):" +msgstr "" + +msgid "We look forward to seeing you on the platform," +msgstr "" + +msgid "The GeoNode team." +msgstr "" + +msgid "is inviting you to join" +msgstr "" + +msgid "To do so, please register at" +msgstr "" + +msgid "that you wish to join her/his group(s) through this link" +msgstr "" + +msgid "is a member of the following group(s)" +msgstr "" + +msgid "Updating Metadata..." +msgstr "" + +msgid "Topic Category is mandatory and cannot be empty!" +msgstr "" + +msgid "Group is mandatory and cannot be empty!" +msgstr "" + +msgid "Identification" +msgstr "" + +msgid "Yes" +msgstr "" + +msgid "Published" +msgstr "" + +msgid "email" +msgstr "" + +msgid "Information" +msgstr "" + +msgid "Identification Image" +msgstr "" + +msgid "Spatial Extent" +msgstr "" + +msgid "Projection System" +msgstr "" + +msgid "Extension x0" +msgstr "" + +msgid "Extension x1" +msgstr "" + +msgid "Extension y0" +msgstr "" + +msgid "Extension y1" +msgstr "" + +msgid "Features" +msgstr "" + +msgid "Contact Points" +msgstr "" + +msgid "References" +msgstr "" + +msgid "Link Online" +msgstr "" + +msgid "Metadata Page" +msgstr "" + +msgid "Online Link" +msgstr "" + +msgid "* Field declared Mandatory by the Metadata Schema" +msgstr "" + +msgid "Please, fill the following items before clicking on preview:" +msgstr "" + +msgid "Request perission" +msgstr "" + +msgid "Request editing for resource: " +msgstr "" + +msgid "Back to resource" +msgstr "" + +msgid "Send request" +msgstr "" + +msgid "Your account has been approved and is now active." +msgstr "" + +msgid "You can use the login form at" +msgstr "" + +msgid "Welcome at" +msgstr "" + +msgid "Your account is now active" +msgstr "" + +msgid "has requested access to the site." +msgstr "" + +msgid "" +"You can enable access by setting the user as active on the admin section" +msgstr "" + +msgid "A user has requested access to the site" +msgstr "" + +msgid "The following document was approved" +msgstr "" + +msgid "owned by" +msgstr "" + +msgid "You can visit the document's detail page here" +msgstr "" + +msgid "A document has been approved" +msgstr "" + +msgid "A comment has been posted for the document" +msgstr "" + +msgid "by the user" +msgstr "" + +msgid "A comment has been posted for a document" +msgstr "" + +msgid "The user" +msgstr "" + +msgid "uploaded the following document" +msgstr "" + +msgid "A document has been uploaded" +msgstr "" + +msgid "The following document was deleted" +msgstr "" + +msgid "A document has been deleted" +msgstr "" + +msgid "The following document was published" +msgstr "" + +msgid "A document has been published" +msgstr "" + +msgid "rated" +msgstr "" + +msgid "the following document" +msgstr "" + +msgid "You can visit the layer's detail page here" +msgstr "" + +msgid "A document has been rated" +msgstr "" + +msgid "The following document was updated" +msgstr "" + +msgid "A document has been updated" +msgstr "" + +msgid "You have received the following notice from" +msgstr "" + +msgid "To change how you receive notifications, please go to " +msgstr "" + +#, python-format +msgid "[%(current_site)s] %(message)s" +msgstr "" + +#, python-format +msgid "%(notice)s" +msgstr "" + +msgid "The following layer was approved" +msgstr "" + +msgid "A layer has been approved" +msgstr "" + +msgid "A comment has been posted for the layer" +msgstr "" + +msgid "A comment has been posted for a layer" +msgstr "" + +msgid "uploaded the following layer" +msgstr "" + +msgid "A new layer has been uploaded" +msgstr "" + +msgid "The following layer was deleted" +msgstr "" + +msgid "A layer has been deleted" +msgstr "" + +msgid "The following layer was published" +msgstr "" + +msgid "A layer has been published" +msgstr "" + +msgid "the following layer" +msgstr "" + +msgid "A layer has been rated" +msgstr "" + +msgid "The following layer was updated" +msgstr "" + +msgid "A layer has been updated" +msgstr "" + +msgid "The following map was approved" +msgstr "" + +msgid "You can visit the map's detail page here" +msgstr "" + +msgid "A map has been approved" +msgstr "" + +msgid "A comment has been posted for the map" +msgstr "" + +msgid "A comment has been posted for a map" +msgstr "" + +msgid "created the following map" +msgstr "" + +msgid "A map has been created" +msgstr "" + +msgid "The following map was deleted" +msgstr "" + +msgid "A map has been deleted" +msgstr "" + +msgid "The following map was published" +msgstr "" + +msgid "A map has been published" +msgstr "" + +msgid "the following map" +msgstr "" + +msgid "A map has been rated" +msgstr "" + +msgid "The following map was updated" +msgstr "" + +msgid "A map has been updated" +msgstr "" + +msgid "Notification Settings" +msgstr "" + +#, python-format +msgid "" +"\n" +"

          \n" +" Note:\n" +" You do not have a verified email address to which notices can be " +"sent. Add one now.\n" +"

          \n" +" " +msgstr "" + +msgid "Notification Type" +msgstr "" + +msgid "requested you to download this resource" +msgstr "" + +msgid "" +"Please go to resource page and assign the download permissions if you wish" +msgstr "" + +msgid "A resource's download has been requested" +msgstr "" + +msgid "Privacy & Cookies Policy" +msgstr "" + +msgid "Privacy & Cookies Policy." +msgstr "" + +msgid "Following" +msgstr "" + +msgid "This user is not following anyone" +msgstr "" + +msgid "Followers" +msgstr "" + +msgid "No one is following this user" +msgstr "" + +msgid "Manage Connections" +msgstr "" + +msgid "Blocking" +msgstr "" + +msgid "Unfollow" +msgstr "" + +msgid "Follow" +msgstr "" + +msgid "Unblock" +msgstr "" + +msgid "Block" +msgstr "" + +msgid "No longer" +msgstr "" + +#, python-format +msgid "You're not %(status_slug)s anyone!" +msgstr "" + +msgid "Success" +msgstr "" + +msgid "You are " +msgstr "" + +msgid "no longer" +msgstr "" + +#, python-format +msgid "%(to_user_username)s's profile" +msgstr "" + +msgid "Responsibles" +msgstr "" + +msgid "Date begins after:" +msgstr "" + +msgid "yyyy-mm-dd" +msgstr "" + +msgid "Date ends before:" +msgstr "" + +msgid "Search by region" +msgstr "" + +msgid "Selection" +msgstr "" + +msgid "No list items selected. Use the selection fields to add." +msgstr "" + +msgid "Filters" +msgstr "" + +msgid "Datasets found" +msgstr "" + +msgid "Maps found" +msgstr "" + +msgid "Documents found" +msgstr "" + +msgid "found" +msgstr "" + +msgid "search" +msgstr "" + +msgid "Search by name" +msgstr "" + +msgid "Total" +msgstr "" + +msgid "Most recent" +msgstr "" + +msgid "Less recent" +msgstr "" + +msgid "A - Z" +msgstr "" + +msgid "Z - A" +msgstr "" + +msgid "Most popular" +msgstr "" + +msgid "Text" +msgstr "" + +msgid "Share This Dataset" +msgstr "" + +msgid "Share This" +msgstr "" + +msgid "Social Network Login Failure" +msgstr "" + +msgid "" +"An error occurred while attempting to login via your social network account." +msgstr "" + +msgid "Code" +msgstr "" + +msgid "Exception" +msgstr "" + +msgid "Account Connections" +msgstr "" + +msgid "" +"You can sign in to your account using any of the following already connected " +"third party accounts:" +msgstr "" + +msgid "" +"You currently have no social network accounts connected to this account." +msgstr "" + +msgid "Add a 3rd Party Account" +msgstr "" + +msgid "Login Cancelled" +msgstr "" + +#, python-format +msgid "" +"You decided to cancel logging in to our site using one of your existing " +"accounts. If this was a mistake, please proceed to sign in." +msgstr "" + +msgid "Signup" +msgstr "" + +msgid "Sign Up" +msgstr "" + +#, python-format +msgid "" +"You are about to use your %(provider_name)s account to login to\n" +"%(site_name)s. As a final step, please complete the following form:" +msgstr "" + +msgid "Sign in with" +msgstr "" + +msgid "Connect with" +msgstr "" + +msgid "Sign up with" +msgstr "" + +msgid "Message Inbox" +msgstr "" + +msgid "Messages" +msgstr "" + +msgid "Mark selected as read" +msgstr "" + +msgid "Mark selected as unread" +msgstr "" + +msgid "Create Message" +msgstr "" + +msgid "To users" +msgstr "" + +msgid "To groups" +msgstr "" + +msgid "Subject" +msgstr "" + +msgid "Content" +msgstr "" + +msgid "Send message" +msgstr "" + +msgid "Back to Inbox" +msgstr "" + +msgid "Sent to" +msgstr "" + +msgid "by" +msgstr "" + +msgid "You have no messages" +msgstr "" + +msgid "Send Reply" +msgstr "" + +msgid "GeoNode Themes Library" +msgstr "" + +msgid "Fill in this section with markdown" +msgstr "" + +msgid "Check this if the jumbotron background image already contains text" +msgstr "" + +msgid "Disabling this slide will hide it from the slide show" +msgstr "" + +msgid "Choose between using jumbotron background and slide show" +msgstr "" + +msgid "Empty content" +msgstr "" + +msgid "Could not access to uploaded data." +msgstr "" + +msgid "" +"One or more XML files was provided, but no matching files were found for " +"them." +msgstr "" + +msgid "" +"One or more SLD files was provided, but no matching files were found for " +"them." +msgstr "" + +msgid "Layer already exists" +msgstr "" + +msgid "Name already in use and overwrite is False" +msgstr "" + +msgid "Error configuring time:" +msgstr "" + +msgid "The GeoServer Import Session is no more available" +msgstr "" + +#, python-brace-format +msgid "Expected to find layer named '{name}' in geoserver" +msgstr "" + +msgid "Exception occurred while parsing the provided Metadata file." +msgstr "" + +msgid "" +"The UUID identifier from the XML Metadata is already in use in this system." +msgstr "" + +msgid "Error configuring Layer" +msgstr "" + +msgid "Invalid zip file detected" +msgstr "" + +msgid "Could not find any valid spatial file inside the uploaded zip" +msgstr "" + +msgid "Invalid kmz file detected" +msgstr "" + +msgid "Could not find any kml files inside the uploaded kmz" +msgstr "" + +msgid "Only one shapefile per zip is allowed" +msgstr "" + +msgid "kml files with more than one GroundOverlay are not supported" +msgstr "" + +msgid "Ground overlay image declared in kml file cannot be found" +msgstr "" + +msgid "Only one kml file per ZIP is allowed" +msgstr "" + +msgid "Only one kml file per kmz is allowed" +msgstr "" + +msgid "" +"You are trying to upload multiple GeoTIFFs without a valid 'indexer." +"properties' file." +msgstr "" + +msgid "Only one raster file per ZIP is allowed" +msgstr "" + +msgid "No multiple rasters allowed" +msgstr "" + +msgid "" +"To support the time step, you must enable the OGC_SERVER DATASTORE option" +msgstr "" + +#, python-brace-format +msgid "Unsupported file type: {e.message}" +msgstr "" + +#, python-brace-format +msgid "File does not exist: {absolute_base_file}" +msgstr "" + +#, python-brace-format +msgid "unknown item state: {task.state}" +msgstr "" + +msgid "error during import: {progress.get('message')}" +msgstr "" + +msgid "Could not find any valid Time Regex for the Mosaic files." +msgstr "" + +msgid "Unsupported DataBase for Mosaics!" +msgstr "" + +msgid "Upload Session invalid or no more accessible!" +msgstr "" + +msgid "Invalid permission level." +msgstr "" + +msgid "Permission Denied" +msgstr "" diff --git a/geonode/locale/al/LC_MESSAGES/djangojs.mo b/geonode/locale/al/LC_MESSAGES/djangojs.mo new file mode 100644 index 0000000..6c5906d Binary files /dev/null and b/geonode/locale/al/LC_MESSAGES/djangojs.mo differ diff --git a/geonode/locale/al/LC_MESSAGES/djangojs.po b/geonode/locale/al/LC_MESSAGES/djangojs.po new file mode 100644 index 0000000..e003c89 --- /dev/null +++ b/geonode/locale/al/LC_MESSAGES/djangojs.po @@ -0,0 +1,556 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-07-08 09:03+0300\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "STORE_UPDATED" +msgstr "" + +#, javascript-format +msgid "Missing a %s file, which is required" +msgstr "" + +msgid "ESRI Shapefile" +msgstr "" + +msgid "ASCII Text File" +msgstr "" + +msgid "GeoTIFF" +msgstr "" + +msgid "Comma Separated Value" +msgstr "" + +msgid "Google Earth KML" +msgstr "" + +msgid "Google Earth KML with a GroundOverlay" +msgstr "" + +msgid "Google Earth KMZ" +msgstr "" + +msgid "GeoJSON" +msgstr "" + +msgid "Zip Archive" +msgstr "" + +msgid "ERDAS Imagine" +msgstr "" + +msgid "National Imagery Transmission Format" +msgstr "" + +msgid "Controlled Image Base-1 Meter" +msgstr "" + +msgid "Controlled Image Base-5 Meter" +msgstr "" + +msgid "Controlled Image Base-10 Meter" +msgstr "" + +msgid "CADRG-Global Navigation Chart" +msgstr "" + +msgid "CADRG-Jet Navigation Chart" +msgstr "" + +msgid "CADRG-Operational Navigation Chart" +msgstr "" + +msgid "CADRG-Tactical Pilotage Chart" +msgstr "" + +msgid "CADRG-Joint Operations Graphic" +msgstr "" + +msgid "CADRG-Topographic Line Map 100K" +msgstr "" + +msgid "CADRG-Topographic Line Map 50K" +msgstr "" + +msgid "JPEG2000" +msgstr "" + +msgid "MrSID-Multi-resolution Seamless Image Database" +msgstr "" + +msgid "XML Metadata File" +msgstr "" + +msgid "Style Layer Descriptor" +msgstr "" + +msgid "JPEG" +msgstr "" + +msgid "PNG" +msgstr "" + +msgid "GIF" +msgstr "" + +msgid "Unexpected error!" +msgstr "" + +msgid "" +" - 400 Bad Request. Server cannot or will not process the request due to " +"something that is perceived to be a client error (e.g., malformed request " +"syntax, invalid request message framing, or deceptive request routing)." +msgstr "" + +msgid "" +" - 401 Unauthorized. Request was not sent with the proper authentication " +"credentials." +msgstr "" + +msgid "" +" - 403 Forbidden. This is generally related to permission rules on your " +"server. Contact the system administrator for more information regarding this " +"error message." +msgstr "" + +msgid "" +" - 404 Not Found. Origin server was unable or unwilling to find the resource " +"requested." +msgstr "" + +msgid "" +" - 405 Method Not Allowed. Origin server is aware of the requested resource, " +"but the request method used is not supported." +msgstr "" + +msgid "" +" - 406 Not Acceptable. Resource is not available at the origin that adheres " +"to negotiation headers that were set prior (e.g. via 'Accept-Charset' and " +"'Accept-Language' headers)." +msgstr "" + +msgid "" +" - 407 Authentication Required. The client did not send the required " +"authentication with the request." +msgstr "" + +msgid "" +" - 408 Request Timeout. The origin server did not receive the complete " +"request in what it considers a reasonable time." +msgstr "" + +msgid "" +" - 409 Conflict. The request did not complete because of a conflict with the " +"current state of the resource. Typically happens on a PUT request where " +"multiple clients are attempting to edit the same resource." +msgstr "" + +msgid "" +" - 410 Gone. The resource requested is permanently missing at the origin." +msgstr "" + +msgid "" +" - 411 Length Required. Client did not define the 'Content-Length' of the " +"request body in the headers and this is required to obtain the resource." +msgstr "" + +msgid "" +" - 412 Precondition Failed. Server denies the request because the resource " +"failed to meet the conditions specified by the client." +msgstr "" + +msgid "" +" - 413 Payload Too Large. Refusal from the server to process the request " +"because the payload sent from the client is larger than the server wished to " +"accept. Server has the optional to close the connection." +msgstr "" + +msgid "" +" - 414 URI Too Long. Refusal from the server that the URI was too long to be " +"processed. For example, if a client is attempting a GET request with an " +"unusually long URI after a POST, this could be seen as a security risk and a " +"414 gets generated." +msgstr "" + +msgid "" +" - 415 Unsupported Media Type. Refusal from the server to process the format " +"of the current payload. One way to identify and fix this issue would be to " +"look at the 'Content-Type' or 'Content-Encoding' headers sent in the " +"client’s request." +msgstr "" + +msgid "" +" - 417 Expectation Failed. Failure of server to meet the requirements " +"specified in the 'Expect' header of the client’s request." +msgstr "" + +msgid "" +" - 429 Too Many Requests. Client has sent too many requests in the specified " +"amount of time according to the server." +msgstr "" + +msgid "" +" - 499 Client Close Request. Nginx specific response code to indicate when " +"the connection has been closed by the client while the server is still " +"processing its request, making server unable to send a status code back." +msgstr "" + +msgid "" +" - 500 Internal Server Error. This error indicates that the server has " +"encountered an unexpected condition. This often occurs when an application " +"request cannot be fulfilled due to the application being configured " +"incorrectly on the server." +msgstr "" + +msgid "" +" - 501 Not Implemented. This error indicates that the HTTP method sent by " +"the client is not supported by the server. This is most often caused by the " +"server being out of date. It is a very rare error and generally requires " +"that the web server be updated." +msgstr "" + +msgid "" +" - 502 Bad Gateway. This error is usually due to improperly configured proxy " +"servers. The first step in resolving the issue is to clear the client's " +"cache." +msgstr "" + +msgid "" +" - 503 Service Unavailable. This error occurs when the server is unable to " +"handle requests due to a temporary overload or due to the server being " +"temporarily closed for maintenance. The error indicates that the server will " +"only temporarily be down." +msgstr "" + +msgid "" +" - 504 Gateway Timeout. GeoNode lost the connection with GeoServer or DB due " +"to a connection timeout. Consider using the management commands to import " +"data!" +msgstr "" + +msgid "" +" - 505 HTTP Version Not Supported. This error occurs when the server refuses " +"to support the HTTP protocol that has been specified by the client computer. " +"This can be caused by the protocol not being specified properly by the " +"client computer; for example, if an invalid version number has been " +"specified." +msgstr "" + +msgid "" +" - 506 Variant Also Negotiates. This error indicates that the server is not " +"properly configured. Contact the system administrator to resolve this issue." +msgstr "" + +msgid "" +" - 507 Insufficient Storage. This error indicates that the server is out of " +"free memory. This is most likely to occur when an application that is being " +"requested cannot allocate the necessary system resources to run. To resolve " +"the issue, the server's hard disk may need to be cleaned of any unnecessary " +"documents to free up more hard disk space, its memory may need to be " +"expanded, or it may simply need to be restarted. Contact the system " +"administrator for more information regarding this error message." +msgstr "" + +msgid "" +" - 509 Bandwidth Limit Exceeded. This error occurs when the bandwidth limit " +"imposed by the system administrator has been reached. The only fix for this " +"issue is to wait until the limit is reset in the following cycle. Consult " +"the system administrator for information about acquiring more bandwidth." +msgstr "" + +msgid "" +" - 510 Not Extended. This error occurs when an extension attached to the " +"HTTP request is not supported by the web server. To resolve the issue, you " +"may need to update the server." +msgstr "" + +msgid "Unknwon" +msgstr "" + +msgid "" +" Error Code. Contact the system administrator for more information regarding " +"this error message." +msgstr "" + +msgid "Additional info: " +msgstr "" + +msgid "Finalizing" +msgstr "" + +msgid "Unexpected Error" +msgstr "" + +msgid "Your " +msgstr "" + +msgid " Info" +msgstr "" + +msgid "Edit Metadata" +msgstr "" + +msgid "Upload Metadata" +msgstr "" + +msgid "Upload SLD" +msgstr "" + +msgid "Manage Styles" +msgstr "" + +msgid "Please wait until GeoNode finished configuring it!" +msgstr "" + +msgid "The column %1 was renamed to %2
          " +msgstr "" + +msgid "Performing Final GeoServer Config Step" +msgstr "" + +msgid "You need to specify more information in order to complete your upload" +msgstr "" + +msgid "Files are ready to be ingested!" +msgstr "" + +msgid "A temporal dimension may be added to this Layer." +msgstr "" + +msgid "Performing GeoServer Config Step" +msgstr "" + +msgid "" +"
          Please check your network connection. In case of Layer Upload make sure " +"GeoServer is running and accepting connections." +msgstr "" + +msgid "
          " +msgstr "" + +msgid "
          Bad request or URL not found." +msgstr "" + +msgid "
          Unknown." +msgstr "" + +msgid "Layer files uploaded, configuring in GeoServer" +msgstr "" + +msgid "Remove" +msgstr "" + +#, javascript-format +msgid "The file %s is an unsupported file type, please select another file." +msgstr "" + +msgid "" +"You are trying to upload an incomplete set of files or not all mandatory " +"options have been validated.\n" +"\n" +"Please check for errors in the form!" +msgstr "" + +#, javascript-format +msgid "Available %s" +msgstr "" + +#, javascript-format +msgid "" +"This is the list of available %s. You may choose some by selecting them in " +"the box below and then clicking the \"Choose\" arrow between the two boxes." +msgstr "" + +#, javascript-format +msgid "Type into this box to filter down the list of available %s." +msgstr "" + +msgid "Filter" +msgstr "" + +msgid "Choose all" +msgstr "" + +#, javascript-format +msgid "Click to choose all %s at once." +msgstr "" + +msgid "Choose" +msgstr "" + +#, javascript-format +msgid "Chosen %s" +msgstr "" + +#, javascript-format +msgid "" +"This is the list of chosen %s. You may remove some by selecting them in the " +"box below and then clicking the \"Remove\" arrow between the two boxes." +msgstr "" + +msgid "Remove all" +msgstr "" + +#, javascript-format +msgid "Click to remove all chosen %s at once." +msgstr "" + +msgid "%(sel)s of %(cnt)s selected" +msgid_plural "%(sel)s of %(cnt)s selected" +msgstr[0] "" +msgstr[1] "" + +msgid "" +"You have unsaved changes on individual editable fields. If you run an " +"action, your unsaved changes will be lost." +msgstr "" + +msgid "" +"You have selected an action, but you haven't saved your changes to " +"individual fields yet. Please click OK to save. You'll need to re-run the " +"action." +msgstr "" + +msgid "" +"You have selected an action, and you haven't made any changes on individual " +"fields. You're probably looking for the Go button rather than the Save " +"button." +msgstr "" + +msgid "'" +msgstr "" + +msgid "-1" +msgstr "" + +msgid "trailingComments" +msgstr "" + +msgid "leadingComments" +msgstr "" + +msgid "innerComments" +msgstr "" + +msgid "Multiselect" +msgstr "" + +msgid "SelectList" +msgstr "" + +msgid "show:" +msgstr "" + +msgid "DraftEditor/root" +msgstr "" + +msgid "DraftEditor/editorContainer" +msgstr "" + +msgid "-" +msgstr "" + +msgid "uirevision" +msgstr "" + +msgid "bgcolor" +msgstr "" + +msgid "bordercolor" +msgstr "" + +msgid "borderwidth" +msgstr "" + +msgid "orientation" +msgstr "" + +msgid "traceorder" +msgstr "" + +msgid "tracegroupgap" +msgstr "" + +msgid "itemsizing" +msgstr "" + +msgid "itemclick" +msgstr "" + +msgid "itemdoubleclick" +msgstr "" + +msgid "x" +msgstr "" + +msgid "xanchor" +msgstr "" + +msgid "y" +msgstr "" + +msgid "yanchor" +msgstr "" + +msgid "valign" +msgstr "" + +msgid "title.text" +msgstr "" + +msgid "title.side" +msgstr "" + +msgid "visible" +msgstr "" + +msgid "thickness" +msgstr "" + +msgid "autorange" +msgstr "" + +msgid "range" +msgstr "" + +msgid "bingroup" +msgstr "" + +msgid "Print configuration broken (" +msgstr "" + +msgid "Print configuration not available (" +msgstr "" + +msgid "go" +msgstr "" + +msgid "goBack" +msgstr "" + +msgid "goForward" +msgstr "" + +msgid "Abort" +msgstr "" + +msgid "As Sibling" +msgstr "" + +msgid "As child" +msgstr "" diff --git a/geonode/locale/am/LC_MESSAGES/django.mo b/geonode/locale/am/LC_MESSAGES/django.mo new file mode 100644 index 0000000..9a9e71b Binary files /dev/null and b/geonode/locale/am/LC_MESSAGES/django.mo differ diff --git a/geonode/locale/am/LC_MESSAGES/django.po b/geonode/locale/am/LC_MESSAGES/django.po new file mode 100644 index 0000000..44478a6 --- /dev/null +++ b/geonode/locale/am/LC_MESSAGES/django.po @@ -0,0 +1,6002 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +msgid "" +msgstr "" +"Project-Id-Version: GeoNode\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-07-08 09:02+0300\n" +"PO-Revision-Date: 2016-08-23 13:57+0000\n" +"Last-Translator: Julien Collaer \n" +"Language-Team: Amharic (http://www.transifex.com/geonode/geonode/language/" +"am/)\n" +"Language: am\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" + +msgid "Request to download a resource" +msgstr "" + +msgid "A request for downloading a resource was sent" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Request resource change" +msgstr "Edit Map Metadata" + +msgid "Owner has requested permissions to modify a resource" +msgstr "" + +msgid "series" +msgstr "series" + +msgid "computer program or routine" +msgstr "" + +msgid "feature type" +msgstr "" + +msgid "copy or imitation of an existing or hypothetical object" +msgstr "" + +msgid "collection hardware" +msgstr "" + +msgid "collection session" +msgstr "" + +msgid "non-geographic data" +msgstr "" + +msgid "property type" +msgstr "" + +msgid "field session" +msgstr "" + +msgid "dataset" +msgstr "" + +msgid "service interfaces" +msgstr "" + +#, fuzzy +#| msgid "distribution description" +msgid "attribute class" +msgstr "distribution description" + +msgid "characteristic of a feature" +msgstr "" + +msgid "tile or spatial subset of geographic data" +msgstr "" + +msgid "feature" +msgstr "" + +msgid "dimension group" +msgstr "" + +msgid "frequency of maintenance for the data is not known" +msgstr "" + +msgid "data is repeatedly and frequently updated" +msgstr "" + +msgid "there are no plans to update the data" +msgstr "" + +msgid "data is updated each day" +msgstr "" + +msgid "data is updated every year" +msgstr "" + +msgid "data is updated as deemed necessary" +msgstr "" + +msgid "data is updated each month" +msgstr "" + +msgid "data is updated every two weeks" +msgstr "" + +msgid "data is updated in intervals that are uneven in duration" +msgstr "" + +msgid "data is updated on a weekly basis" +msgstr "" + +msgid "data is updated twice each year" +msgstr "" + +msgid "data is updated every three months" +msgstr "" + +msgid "No information provided" +msgstr "" + +msgid "Category is required." +msgstr "" + +msgid "Keywords from Thesaurus" +msgstr "" + +msgid "List of keywords from Thesaurus" +msgstr "" + +msgid "Abstract" +msgstr "" + +msgid "Purpose" +msgstr "" + +#, fuzzy +#| msgid "Notice settings" +msgid "Other constraints" +msgstr "Notice settings" + +msgid "Supplemental information" +msgstr "" + +msgid "Data quality statement" +msgstr "" + +msgid "Owner" +msgstr "" + +msgid "Date" +msgstr "" + +msgid "temporal extent start" +msgstr "" + +msgid "temporal extent end" +msgstr "" + +msgid "Person outside GeoNode (fill form)" +msgstr "" + +msgid "Point of Contact" +msgstr "" + +msgid "Metadata Author" +msgstr "" + +msgid "Free-text Keywords" +msgstr "" + +msgid "" +"A space or comma-separated list of keywords. Use the widget to select from " +"Hierarchical tree." +msgstr "" + +msgid "Regions" +msgstr "" + +msgid "Group" +msgstr "" + +msgid "Category" +msgstr "" + +msgid "License" +msgstr "" + +msgid "Language" +msgstr "" + +msgid "User" +msgid_plural "Users" +msgstr[0] "" +msgstr[1] "" + +msgid "Permission Type" +msgstr "" + +msgid "Mode" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Resource" +msgstr "Edit Map Metadata" + +msgid "Reason" +msgstr "" + +msgid "Short reasoning behind the request" +msgstr "" + +msgid "function performed by the responsible party" +msgstr "" + +msgid "brief narrative summary of the content of the resource(s)" +msgstr "" + +msgid "reference date for the cited resource" +msgstr "" + +msgid "identification of when a given event occurred" +msgstr "" + +msgid "version of the cited resource" +msgstr "" + +msgid "" +"authority or function assigned, as to a ruler, legislative assembly, " +"delegate, or the like." +msgstr "" + +msgid "a DOI will be added by Admin before publication." +msgstr "" + +msgid "summary of the intentions with which the resource(s) was developed" +msgstr "" + +msgid "" +"frequency with which modifications and deletions are made to the data after " +"it is first produced" +msgstr "" + +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated)" +msgstr "" + +msgid "" +"formalised word(s) or phrase(s) from a fixed thesaurus used to describe the " +"subject (space or comma-separated)" +msgstr "" + +msgid "keyword identifies a location" +msgstr "" + +msgid "limitation(s) placed upon the access or use of the data." +msgstr "" + +msgid "" +"other restrictions and legal prerequisites for accessing and using the " +"resource or metadata" +msgstr "" + +msgid "license of the dataset" +msgstr "" + +msgid "language used within the dataset" +msgstr "" + +msgid "" +"high-level geographic data thematic classification to assist in the grouping " +"and search of available geographic data sets." +msgstr "" + +msgid "method used to represent geographic information in the dataset." +msgstr "" + +msgid "time period covered by the content of the dataset (start)" +msgstr "" + +msgid "time period covered by the content of the dataset (end)" +msgstr "" + +msgid "" +"general explanation of the data producer's knowledge about the lineage of a " +"dataset" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "title" +msgstr "Edit Map Metadata" + +msgid "name by which the cited resource is known" +msgstr "" + +msgid "abstract" +msgstr "" + +msgid "purpose" +msgstr "" + +msgid "date" +msgstr "" + +msgid "date type" +msgstr "" + +msgid "edition" +msgstr "" + +#, fuzzy +#| msgid "distribution URL" +msgid "Attribution" +msgstr "distribution URL" + +msgid "DOI" +msgstr "" + +msgid "maintenance frequency" +msgstr "" + +msgid "keywords" +msgstr "" + +msgid "keywords region" +msgstr "" + +#, fuzzy +#| msgid "Your selections" +msgid "restrictions" +msgstr "Your selections" + +msgid "restrictions other" +msgstr "" + +msgid "language" +msgstr "" + +msgid "spatial representation type" +msgstr "" + +msgid "supplemental information" +msgstr "" + +msgid "any other descriptive information about the dataset" +msgstr "" + +msgid "data quality statement" +msgstr "" + +msgid "CSW typename" +msgstr "" + +msgid "CSW schema" +msgstr "" + +msgid "CSW source" +msgstr "" + +msgid "CSW insert date" +msgstr "" + +msgid "CSW type" +msgstr "" + +msgid "CSW anytext" +msgstr "" + +msgid "CSW WKT geometry" +msgstr "" + +msgid "Metadata uploaded preserve" +msgstr "" + +msgid "Featured" +msgstr "" + +msgid "Should this resource be advertised in home page?" +msgstr "" + +msgid "Is Published" +msgstr "" + +msgid "Should this resource be published and searchable?" +msgstr "" + +msgid "Approved" +msgstr "" + +msgid "Is this resource validated from a publisher or editor?" +msgstr "" + +#, fuzzy +#| msgid "Thumbnail" +msgid "Thumbnail url" +msgstr "Thumbnail" + +msgid "State" +msgstr "" + +msgid "Hold the resource processing state." +msgstr "" + +msgid "Dirty State" +msgstr "" + +msgid "Security Rules Are Not Synched with GeoServer!" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Resource Type" +msgstr "Edit Map Metadata" + +msgid "Metadata" +msgstr "" + +msgid "If true, will be excluded from search" +msgstr "" + +msgid "For example \"kml\"" +msgstr "" + +msgid "For example \"View in Google Earth\"" +msgstr "" + +msgid "For example \"text/xml\"" +msgstr "" + +msgid "About" +msgstr "" + +msgid "Responsible, Point of Contact, Metadata Author" +msgstr "" + +msgid "Responsible" +msgstr "" + +msgid "Title" +msgstr "" + +msgid "SRID" +msgstr "" + +msgid "For more info see" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Creation Date" +msgstr "Create a map" + +msgid "Publication Date" +msgstr "" + +msgid "Revision Date" +msgstr "" + +msgid "Type" +msgstr "" + +msgid "Keywords" +msgstr "" + +msgid "More info" +msgstr "" + +msgid "Maintenance Frequency" +msgstr "" + +msgid "Restrictions" +msgstr "" + +msgid "Edition" +msgstr "" + +msgid "Temporal Extent" +msgstr "" + +msgid "Data Quality" +msgstr "" + +msgid "Supplemental Information" +msgstr "" + +msgid "Spatial Representation Type" +msgstr "" + +msgid "No content created yet." +msgstr "" + +msgid "Temporal Serie" +msgstr "" + +msgid "Select" +msgstr "" + +msgid "Service is" +msgstr "" + +msgid "online" +msgstr "" + +msgid "offline" +msgstr "" + +msgid "SECURITY NOT YET SYNCHRONIZED" +msgstr "" + +msgid "Sync permissions immediately" +msgstr "" + +msgid "PENDING APPROVAL" +msgstr "" + +msgid "UNPUBLISHED" +msgstr "" + +msgid "Create a Map" +msgstr "" + +msgid "View Map" +msgstr "" + +msgid "Batch Edit" +msgstr "" + +msgid "Cancel" +msgstr "" + +msgid "Submit" +msgstr "" + +msgid "Set Permissions" +msgstr "" + +msgid "Discover the available datasets." +msgstr "" + +#, fuzzy +#| msgid "Set Map Thumbnail" +msgid "Upload a Thumbnail" +msgstr "Set Map Thumbnail" + +msgid "Remove (and use auto generated thumbnail)" +msgstr "" + +msgid "Upload" +msgstr "" + +#, fuzzy +#| msgid "Remote Thumbnail" +msgid "Auto generated thumbnail" +msgstr "Remote Thumbnail" + +#, fuzzy +#| msgid "Create a map" +msgid "Raster Layer" +msgstr "Create a map" + +#, fuzzy +#| msgid "Create a map" +msgid "Vector Layer" +msgstr "Create a map" + +msgid "Vector Temporal Serie" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Remote Layer" +msgstr "Create a map" + +#, fuzzy +#| msgid "Create a map" +msgid "WMS Cascade Layer" +msgstr "Create a map" + +msgid "me" +msgstr "እኔ" + +msgid "System message: A request to modify resource" +msgstr "" + +msgid "The resource owner has requested to modify the resource" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Resource title" +msgstr "Edit Map Metadata" + +msgid "Reason for the request" +msgstr "" + +msgid "" +"To allow the change, set the resource to not \"Approved\" under the metadata " +"settingsand write message to the owner to notify him" +msgstr "" + +msgid "Resource Cloned Successfully!" +msgstr "" + +#, python-brace-format +msgid "Error Occurred while Cloning the Resource: {e}" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Resource Metadata" +msgstr "Edit Map Metadata" + +msgid "Thumbnail" +msgstr "Thumbnail" + +msgid "Resource ID" +msgstr "" + +msgid "Start" +msgstr "" + +msgid "End" +msgstr "" + +msgid "Extent" +msgstr "" + +msgid "Spatial Reference System Identifier" +msgstr "" + +msgid "Attributes" +msgstr "" + +msgid "Attribute name" +msgstr "" + +msgid "Description" +msgstr "" + +msgid "GeoNode Client Library" +msgstr "" + +msgid "" +"Flora and/or fauna in natural environment. Examples: wildlife, vegetation, " +"biological sciences, ecology, wilderness, sealife, wetlands, habitat." +msgstr "" + +msgid "Biota" +msgstr "" + +msgid "" +"Legal land descriptions. Examples: political and administrative boundaries." +msgstr "" + +msgid "Boundaries" +msgstr "" + +msgid "" +"Processes and phenomena of the atmosphere. Examples: cloud cover, weather, " +"climate, atmospheric conditions, climate change, precipitation." +msgstr "" + +msgid "Climatology Meteorology Atmosphere" +msgstr "" + +msgid "" +"Economic activities, conditions and employment. Examples: production, " +"labour, revenue, commerce, industry, tourism and ecotourism, forestry, " +"fisheries, commercial or subsistence hunting, exploration and exploitation " +"of resources such as minerals, oil and gas." +msgstr "" + +msgid "Economy" +msgstr "" + +msgid "" +"Height above or below sea level. Examples: altitude, bathymetry, digital " +"elevation models, slope, derived products." +msgstr "" + +msgid "Elevation" +msgstr "" + +msgid "" +"Environmental resources, protection and conservation. Examples: " +"environmental pollution, waste storage and treatment, environmental impact " +"assessment, monitoring environmental risk, nature reserves, landscape." +msgstr "" + +msgid "Environment" +msgstr "" + +msgid "" +"Rearing of animals and/or cultivation of plants. Examples: agriculture, " +"irrigation, aquaculture, plantations, herding, pests and diseases affecting " +"crops and livestock." +msgstr "" + +msgid "Farming" +msgstr "" + +msgid "" +"Information pertaining to earth sciences. Examples: geophysical features and " +"processes, geology, minerals, sciences dealing with the composition, " +"structure and origin of the earth s rocks, risks of earthquakes, volcanic " +"activity, landslides, gravity information, soils, permafrost, hydrogeology, " +"erosion." +msgstr "" + +msgid "Geoscientific Information" +msgstr "" + +msgid "" +"Health, health services, human ecology, and safety. Examples: disease and " +"illness, factors affecting health, hygiene, substance abuse, mental and " +"physical health, health services." +msgstr "" + +msgid "Health" +msgstr "" + +msgid "" +"Base maps. Examples: land cover, topographic maps, imagery, unclassified " +"images, annotations." +msgstr "" + +msgid "Imagery Base Maps Earth Cover" +msgstr "" + +msgid "" +"Inland water features, drainage systems and their characteristics. Examples: " +"rivers and glaciers, salt lakes, water utilization plans, dams, currents, " +"floods, water quality, hydrographic charts." +msgstr "" + +msgid "Inland Waters" +msgstr "" + +msgid "" +"Military bases, structures, activities. Examples: barracks, training " +"grounds, military transportation, information collection." +msgstr "" + +msgid "Intelligence Military" +msgstr "" + +msgid "" +"Positional information and services. Examples: addresses, geodetic networks, " +"control points, postal zones and services, place names." +msgstr "" + +msgid "Location" +msgstr "" + +msgid "" +"Features and characteristics of salt water bodies (excluding inland waters). " +"Examples: tides, tidal waves, coastal information, reefs." +msgstr "" + +msgid "Oceans" +msgstr "" + +msgid "" +"Information used for appropriate actions for future use of the land. " +"Examples: land use maps, zoning maps, cadastral surveys, land ownership." +msgstr "" + +msgid "Planning Cadastre" +msgstr "" + +msgid "" +"Settlements, anthropology, archaeology, education, traditional beliefs, " +"manners and customs, demographic data, recreational areas and activities, " +"social impact assessments, crime and justice, census information. Economic " +"activities, conditions and employment." +msgstr "" + +msgid "Population" +msgstr "" + +msgid "" +"Characteristics of society and cultures. Examples: settlements, " +"anthropology, archaeology, education, traditional beliefs, manners and " +"customs, demographic data, recreational areas and activities, social impact " +"assessments, crime and justice, census information." +msgstr "" + +msgid "Society" +msgstr "" + +msgid "" +"Man-made construction. Examples: buildings, museums, churches, factories, " +"housing, monuments, shops, towers." +msgstr "" + +msgid "Structure" +msgstr "" + +msgid "" +"Means and aids for conveying persons and/or goods. Examples: roads, airports/" +"airstrips, shipping routes, tunnels, nautical charts, vehicle or vessel " +"location, aeronautical charts, railways." +msgstr "" + +msgid "Transportation" +msgstr "" + +msgid "" +"Energy, water and waste systems and communications infrastructure and " +"services. Examples: hydroelectricity, geothermal, solar and nuclear sources " +"of energy, water purification and distribution, sewage collection and " +"disposal, electricity and gas distribution, data communication, " +"telecommunication, radio, communication networks." +msgstr "" + +msgid "Utilities Communication" +msgstr "" + +msgid "Document Created" +msgstr "" + +msgid "A Document was created" +msgstr "" + +msgid "Document Updated" +msgstr "" + +msgid "A Document was updated" +msgstr "" + +msgid "Document Approved" +msgstr "" + +msgid "A Document was approved by a Manager" +msgstr "" + +msgid "Document Published" +msgstr "" + +msgid "A Document was published" +msgstr "" + +msgid "Document Deleted" +msgstr "" + +msgid "A Document was deleted" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Comment on Document" +msgstr "Create a map" + +msgid "A Document was commented on" +msgstr "" + +msgid "Rating for Document" +msgstr "ለሰነድ የተሰጠ ደረጃ" + +msgid "A rating was given to a document" +msgstr "" + +msgid "Width" +msgstr "" + +msgid "Height" +msgstr "" + +msgid "Make" +msgstr "" + +msgid "Model" +msgstr "" + +msgid "Latitude" +msgstr "" + +msgid "Not specified" +msgstr "" + +msgid "Longitude" +msgstr "" + +msgid "Flash" +msgstr "" + +msgid "True" +msgstr "" + +msgid "False" +msgstr "" + +msgid "Speed Rating" +msgstr "" + +msgid "Link to" +msgstr "" + +msgid "File" +msgstr "" + +msgid "Document must be a file or url." +msgstr "" + +msgid "A document cannot have both a file and a url." +msgstr "" + +msgid "This file type is not allowed" +msgstr "" + +msgid "Permissions must be valid JSON." +msgstr "" + +msgid "The URL of the document if it is external." +msgstr "" + +msgid "URL" +msgstr "" + +msgid "Your browser does not support the audio element." +msgstr "" + +msgid "Your browser does not support the video tag." +msgstr "" + +msgid "Download the" +msgstr "" + +msgid "document" +msgstr "" + +msgid "External Resource" +msgstr "" + +msgid "Rate this document" +msgstr "" + +msgid "Average Rating" +msgstr "" + +msgid "Metadata Detail" +msgstr "" + +msgid "Download Document" +msgstr "" + +msgid "Request Download" +msgstr "" + +msgid "Edit Document" +msgstr "" + +msgid "Request change" +msgstr "" + +msgid "Wizard" +msgstr "" + +msgid "Advanced Edit" +msgstr "" + +msgid "Document" +msgid_plural "Documents" +msgstr[0] "" +msgstr[1] "" + +msgid "Clone" +msgstr "" + +msgid "Replace" +msgstr "" + +msgid "Remove" +msgstr "" + +msgid "Close" +msgstr "" + +msgid "Download Metadata" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Full metadata" +msgstr "Edit Map Metadata" + +msgid "Text format" +msgstr "" + +msgid "HTML format" +msgstr "" + +msgid "Standard Metadata - XML format" +msgstr "" + +#, fuzzy +#| msgid "Replace this Document" +msgid "Resources using this document" +msgstr "Replace this Document" + +msgid "List of resources using this document:" +msgstr "" + +msgid "This document is not related to any maps or layers" +msgstr "" + +msgid "Permissions" +msgstr "" + +msgid "Click the button below to change the permissions of this document." +msgstr "" + +msgid "Change Document Permissions" +msgstr "" + +msgid "Explore Documents" +msgstr "" + +msgid "Upload Documents" +msgstr "" + +#, python-format +msgid "for %(document_title)s" +msgstr "" + +msgid "Completeness" +msgstr "" + +msgid "Metadata Schema mandatory fields completed" +msgstr "" + +msgid "Check Schema mandatory fields" +msgstr "" + +msgid "Error updating metadata. Please check the following fields: " +msgstr "" + +msgid "Done" +msgstr "" + +msgid "Metadata Provider" +msgstr "" + +msgid "<< Back" +msgstr "" + +msgid "Update" +msgstr "" + +msgid "Next >>" +msgstr "" + +msgid "Select an option" +msgstr "" + +msgid "Edit Metadata" +msgstr "" + +msgid "Editing details for" +msgstr "" + +msgid "Return to Document" +msgstr "" + +msgid "Deleting" +msgstr "" + +msgid "Remove Document" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(document_title)s?\n" +" " +msgstr "" + +msgid "Yes, I am sure" +msgstr "" + +msgid "No, don't remove it" +msgstr "" + +msgid "Replace Document" +msgstr "" + +msgid "Replace " +msgstr "" + +msgid "Clear" +msgstr "" + +msgid "Upload Document's Thumbnail" +msgstr "" + +msgid "Drop files here" +msgstr "" + +msgid " or select them one by one:" +msgstr "" + +msgid "Choose Files" +msgstr "" + +msgid "Files to be uploaded" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Is Upload Metadata XML Form" +msgstr "Edit Map Metadata" + +msgid "Upload files" +msgstr "" + +msgid "Upload Document" +msgstr "" + +msgid "Allowed document types" +msgstr "" + +msgid "Done!" +msgstr "" + +msgid "Edit" +msgstr "" + +#, fuzzy +#| msgid "Notice settings" +msgid "Settings" +msgstr "Notice settings" + +msgid "Mandatory" +msgstr "" + +msgid "Optional" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Basic Metadata" +msgstr "Edit Map Metadata" + +msgid "Location and Licenses" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Required Metadata" +msgstr "Edit Map Metadata" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Optional Metadata" +msgstr "Edit Map Metadata" + +msgid "Choose one of the following..." +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Other, Optional, Metadata" +msgstr "Edit Map Metadata" + +msgid "Responsible Parties" +msgstr "" + +msgid "Responsible and Permissions" +msgstr "" + +msgid "Publishing" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "You are not allowed to view this document." +msgstr "You do not have permission to view this document" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "You are not permitted to delete this document" +msgstr "You do not have permission to view this document" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "You do not have permissions for this document." +msgstr "You do not have permission to view this document" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "You are not permitted to modify this document" +msgstr "You do not have permission to view this document" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "You are not permitted to modify this document's metadata" +msgstr "You do not have permission to view this document" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "You are not permitted to view this document" +msgstr "You do not have permission to view this document" + +msgid "Not allowed" +msgstr "" + +msgid "Not found" +msgstr "" + +msgid "You must set a point of contact for this resource" +msgstr "" + +msgid "You must set an author for this resource" +msgstr "" + +msgid "Favorite" +msgstr "" + +msgid "Add to Favorites" +msgstr "Add to Favorites" + +#, fuzzy +#| msgid "Add to Favorites" +msgid "Go to Favorites" +msgstr "Add to Favorites" + +msgid "Log in to add/delete Favorites." +msgstr "" + +msgid "Delete from Favorites" +msgstr "" + +msgid "Favorites" +msgstr "" + +msgid "Favorites for" +msgstr "" + +msgid "Item" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "App Created" +msgstr "Edit Map Metadata" + +msgid "A App was created" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "App Updated" +msgstr "Edit Map Metadata" + +msgid "A App was updated" +msgstr "" + +msgid "App Approved" +msgstr "" + +msgid "A App was approved by a Manager" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "App Published" +msgstr "Edit Map Metadata" + +msgid "A App was published" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "App Deleted" +msgstr "Edit Map Metadata" + +msgid "A App was deleted" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Comment on App" +msgstr "Create a map" + +msgid "An App was commented on" +msgstr "" + +#, fuzzy +#| msgid "Rating for Document" +msgid "Rating for App" +msgstr "ለሰነድ የተሰጠ ደረጃ" + +msgid "A rating was given to an App" +msgstr "" + +msgid "Name" +msgstr "" + +msgid "zoom" +msgstr "" + +#, fuzzy +#| msgid "Your selections" +msgid "projection" +msgstr "Your selections" + +msgid "center X" +msgstr "" + +msgid "center Y" +msgstr "" + +msgid "Site URL" +msgstr "" + +#, python-format +msgid "%(resource.type)s" +msgstr "" + +msgid "Rate this" +msgstr "" + +msgid "Editing Tools" +msgstr "" + +msgid "Set" +msgstr "" + +msgid "View" +msgstr "View" + +msgid "Specify which users can view or modify this map" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "Change Permissions of this" +msgstr "You do not have permission to view this map" + +msgid "Embed Iframe" +msgstr "" + +msgid "" +"To embed this map, add the following code snippet and customize its " +"properties (scrolling, width, height) based on your needs to your site" +msgstr "" + +msgid "Download" +msgstr "" + +#, python-format +msgid "%(GEONODE_APPS_NAME)s" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Create New" +msgstr "Create a map" + +msgid "Explore" +msgstr "" + +#, python-format +msgid "for %(map_title)s" +msgstr "" + +msgid "" +"Note: this geoapp's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" + +msgid "Explore Apps" +msgstr "" + +#, python-format +msgid "" +"\n" +" Editing details for %(map_title)s\n" +" " +msgstr "" + +msgid "Return to GeoApp" +msgstr "" + +msgid "New Map" +msgstr "" + +#, fuzzy, python-format +#| msgid "" +#| "\n" +#| " Are you sure you want to remove %(document_title)s?\n" +#| " " +msgid "" +"\n" +" Are you sure you want to remove %(resource_title)s?\n" +" " +msgstr "" +"\n" +" Are you sure you want to remove %(document_title)s?\n" +" " + +#, fuzzy +#| msgid "Notice settings" +msgid "Other Settings" +msgstr "Notice settings" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You are not permitted to delete this app." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You do not have permissions for this app." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You must be logged in to save this app" +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You are not permitted to save or edit this app." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not allowed to modify this app's metadata." +msgstr "You do not have permission to view this layer" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You are not allowed to view this app." +msgstr "You do not have permission to view this map" + +msgid "An unknown error has occured." +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Layer Uploaded" +msgstr "Edit Map Metadata" + +msgid "A layer was uploaded" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Comment on Layer" +msgstr "Create a map" + +msgid "A layer was commented on" +msgstr "" + +msgid "Rating for Layer" +msgstr "" + +msgid "A rating was given to a layer" +msgstr "" + +msgid "Points" +msgstr "" + +msgid "Lines" +msgstr "" + +msgid "Polygons" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Dataset name" +msgstr "Edit Map Metadata" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Dataset title" +msgstr "Edit Map Metadata" + +msgid "Geometry type" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Create Dataset" +msgstr "Create a map" + +msgid "Explore Datasets" +msgstr "" + +msgid "Create an empty dataset" +msgstr "" + +#, fuzzy +#| msgid "Add to Favorites" +msgid "Add Attribute" +msgstr "Add to Favorites" + +msgid "Create" +msgstr "" + +msgid "String" +msgstr "" + +msgid "Integer" +msgstr "" + +msgid "Float" +msgstr "" + +msgid "No title provided" +msgstr "" + +msgid "No abstract provided" +msgstr "" + +msgid "An exception occurred loading data to PostGIS" +msgstr "" + +msgid " Additionally an error occured during database cleanup" +msgstr "" + +msgid "Invalid Projection. Layer is missing CRS!" +msgstr "" + +msgid "GZIP" +msgstr "" + +msgid "GeoTIFF" +msgstr "" + +msgid "Zipped Shapefile" +msgstr "" + +msgid "GML 2.0" +msgstr "" + +msgid "GML 3.1.1" +msgstr "" + +msgid "CSV" +msgstr "" + +msgid "Excel" +msgstr "" + +msgid "GeoJSON" +msgstr "" + +msgid "JPEG" +msgstr "" + +msgid "PDF" +msgstr "" + +msgid "PNG" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You don't have permissions to change style for this layer" +msgstr "You do not have permission to view this layer" + +msgid "Bad HTTP Authorization Credentials." +msgstr "" + +msgid "" +"a short version of the name consisting only of letters, numbers, underscores " +"and hyphens." +msgstr "" + +msgid "A group already exists with that slug." +msgstr "" + +msgid "A group already exists with that name." +msgstr "" + +msgid "User Identifiers" +msgstr "" + +msgid "Assign manager role" +msgstr "" + +#, python-format +msgid "" +"The following are not valid usernames: %(errors)s; not added to the group" +msgstr "" + +#, fuzzy +#| msgid "Group details" +msgid "Group Categories" +msgstr "Group details" + +msgid "Public" +msgstr "" + +msgid "Public (invite-only)" +msgstr "" + +msgid "Private" +msgstr "" + +msgid "" +"Public: Any registered user can view and join a public group.
          Public " +"(invite-only):Any registered user can view the group. Only invited users " +"can join.
          Private: Registered users cannot see any details about the " +"group, including membership. Only invited users can join." +msgstr "" + +msgid "" +"Email used to contact one or all group members, such as a mailing list, " +"shared email, or exchange group." +msgstr "" + +msgid "Logo" +msgstr "" + +msgid "Email" +msgstr "" + +msgid "A space or comma-separated list of keywords" +msgstr "" + +msgid "Access" +msgstr "" + +msgid "Categories" +msgstr "" + +msgid "Manager" +msgstr "" + +msgid "Member" +msgstr "" + +msgid "Activity Feed for" +msgstr "" + +msgid "All" +msgstr "" + +msgid "Layers" +msgstr "" + +msgid "Maps" +msgstr "" + +msgid "Documents" +msgstr "" + +msgid "Comments" +msgstr "" + +msgid "No actions yet" +msgstr "" + +msgid "Explore Group Categories" +msgstr "" + +msgid "Create a New Group Category" +msgstr "" + +msgid "Explore Groups" +msgstr "" + +msgid "Create a Group" +msgstr "" + +msgid "Update Group" +msgstr "" + +msgid "Remove Group" +msgstr "" + +msgid "Create Group" +msgstr "" + +msgid "Last Modified" +msgstr "" + +msgid "This group has not created a logo." +msgstr "" + +msgid "Edit Group Details" +msgstr "" + +msgid "Manage Group Members" +msgstr "" + +msgid "Delete this Group" +msgstr "" + +msgid "Group Activities" +msgstr "" + +#, python-format +msgid "" +"\n" +" This group is %(access)s.\n" +" " +msgstr "" + +msgid "Anyone may join this group." +msgstr "" + +msgid "Join Group" +msgstr "" + +msgid "Anyone may view this group but membership is by invitation only." +msgstr "" + +msgid "Membership is by invitation only." +msgstr "" + +msgid "Managers" +msgstr "" + +msgid "Members" +msgstr "" + +msgid "Create a New Group" +msgstr "" + +msgid "Edit Members for" +msgstr "" + +msgid "Current Members" +msgstr "" + +msgid "Promote" +msgstr "" + +msgid "Demote" +msgstr "" + +msgid "Role" +msgstr "" + +msgid "Add new members" +msgstr "" + +msgid "Add Group Members" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove the group: %(group_title)s?\n" +" " +msgstr "" + +msgid "Remove Maps/Data from Group" +msgstr "" + +msgid "Remove selected maps/data" +msgstr "" + +#, fuzzy +#| msgid "Group details" +msgid "Edit Group Category Details" +msgstr "Group details" + +msgid "Groups" +msgstr "" + +msgid "Create Group Category" +msgstr "" + +msgid "Create a Group Category" +msgstr "" + +#, fuzzy +#| msgid "Group details" +msgid "Edit Group Category" +msgstr "Group details" + +msgid "Save" +msgstr "" + +msgid "ready" +msgstr "" + +msgid "updating-harvestable-resources" +msgstr "" + +msgid "harvesting-resources" +msgstr "" + +msgid "checking-availability" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Harvester name" +msgstr "Edit Map Metadata" + +msgid "Base URL of the remote service that is to be harvested" +msgstr "" + +msgid "" +"Whether to periodically schedule this harvester to look for resources on the " +"remote service" +msgstr "" + +msgid "" +"How often (in minutes) should new harvesting sessions be automatically " +"scheduled? Setting this value to zero has the same effect as setting " +"`scheduling_enabled` to False " +msgstr "" + +msgid "Whether the remote service is known to be available or not" +msgstr "" + +msgid "" +"How often (in minutes) should the remote service be checked for availability?" +msgstr "" + +msgid "Last time the remote server was checked for availability" +msgstr "" + +msgid "Last time the remote server was checked for harvestable resources" +msgstr "" + +msgid "Default owner of harvested resources" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "Default access permissions of harvested resources" +msgstr "You do not have permission to view this layer" + +msgid "" +"Should new resources be harvested automatically without explicit selection?" +msgstr "" + +msgid "" +"Orphan resources are those that have previously been created by means of a " +"harvesting operation but that GeoNode can no longer find on the remote " +"service being harvested. Should these resources be deleted from GeoNode " +"automatically? This also applies to when a harvester configuration is " +"deleted, in which case all of the resources that originated from that " +"harvester are now considered to be orphan." +msgstr "" + +msgid "Date of last update to the harvester configuration." +msgstr "" + +msgid "" +"Harvester class used to perform harvesting sessions. New harvester types can " +"be added by an admin by changing the main GeoNode `settings.py` file" +msgstr "" + +msgid "" +"Configuration specific to each harvester type. Please consult GeoNode " +"documentation on harvesting for more info. This field is mandatory, so at " +"the very least an empty object (i.e. {}) must be supplied." +msgstr "" + +msgid "Periodic task used to configure harvest scheduling" +msgstr "" + +msgid "Periodic task used to check availability of the remote" +msgstr "" + +#, python-brace-format +msgid "Check availability of {self.name}" +msgstr "" + +msgid "updating-harvestable-resource" +msgstr "" + +msgid "harvesting-resource" +msgstr "" + +msgid "" +"Identifier that allows referencing the resource on its remote service in a " +"unique fashion. This is usually automatically filled by the harvester " +"worker. The harvester worker needs to know how to either read or generate " +"this from each remote resource in order to be able to compare the " +"availability of resources between consecutive harvesting sessions." +msgstr "" + +msgid "" +"Type of the resource in the remote service. Each harvester worker knows how " +"to fill this field, in accordance with the resources for which harvesting is " +"supported" +msgstr "" + +#, python-brace-format +msgid "The e-mail address '{email}' has already been invited." +msgstr "" + +#, python-brace-format +msgid "The e-mail address '{email}' has already accepted an invite." +msgstr "" + +#, python-brace-format +msgid "An active user is already using the e-mail address '{email}'" +msgstr "" + +msgid "E-mail" +msgstr "" + +#, python-format +msgid "Invitations successfully sent to '%(email)s'" +msgstr "" + +#, python-format +msgid "" +"Sorry, it was not possible to invite '%(email)s' due to the following isse: " +"%(error)s (%(type)s)" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Layer Created" +msgstr "Edit Map Metadata" + +msgid "A Layer was created" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Layer Updated" +msgstr "Edit Map Metadata" + +msgid "A Layer was updated" +msgstr "" + +msgid "Layer Approved" +msgstr "" + +msgid "A Layer was approved by a Manager" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Layer Published" +msgstr "Edit Map Metadata" + +msgid "A Layer was published" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Layer Deleted" +msgstr "Edit Map Metadata" + +msgid "A Layer was deleted" +msgstr "" + +msgid "YYYYMMDD" +msgstr "" + +msgid "YYYYMMDD'T'hhmmss" +msgstr "" + +msgid "YYYYMMDD'T'hhmmss'Z'" +msgstr "" + +msgid "style name" +msgstr "" + +msgid "sld text" +msgstr "" + +msgid "sld version" +msgstr "" + +msgid "sld url" +msgstr "" + +msgid "Workspace" +msgstr "" + +msgid "Store" +msgstr "" + +msgid "Storetype" +msgstr "" + +msgid "Typename" +msgstr "" + +msgid "Is mosaic?" +msgstr "" + +msgid "Has time?" +msgstr "" + +msgid "Has elevation?" +msgstr "" + +msgid "Time regex" +msgstr "" + +msgid "Elevation regex" +msgstr "" + +msgid "use featureinfo custom template?" +msgstr "" + +msgid "specifies wether or not use a custom GetFeatureInfo template." +msgstr "" + +msgid "featureinfo custom template" +msgstr "" + +msgid "the custom GetFeatureInfo template HTML contents." +msgstr "" + +msgid "File cannot be opened, maybe check the encoding" +msgstr "" + +#, fuzzy +#| msgid "distribution description" +msgid "attribute name" +msgstr "distribution description" + +msgid "name of attribute as stored in shapefile/spatial database" +msgstr "" + +#, fuzzy +#| msgid "distribution description" +msgid "attribute description" +msgstr "distribution description" + +msgid "description of attribute to be used in metadata" +msgstr "" + +#, fuzzy +#| msgid "distribution description" +msgid "attribute label" +msgstr "distribution description" + +msgid "title of attribute as displayed in GeoNode" +msgstr "" + +#, fuzzy +#| msgid "distribution description" +msgid "attribute type" +msgstr "distribution description" + +msgid "the data type of the attribute (integer, string, geometry, etc)" +msgstr "" + +msgid "visible?" +msgstr "" + +msgid "specifies if the attribute should be displayed in identify results" +msgstr "" + +msgid "display order" +msgstr "" + +msgid "" +"specifies the order in which attribute should be displayed in identify " +"results" +msgstr "" + +msgid "Label" +msgstr "" + +msgid "Image" +msgstr "" + +msgid "Video (mp4)" +msgstr "" + +msgid "Video (ogg)" +msgstr "" + +msgid "Video (webm)" +msgstr "" + +msgid "Video (3gp)" +msgstr "" + +msgid "Video (flv)" +msgstr "" + +msgid "Video (YouTube/VIMEO - embedded)" +msgstr "" + +msgid "Audio" +msgstr "" + +msgid "IFRAME" +msgstr "" + +#, fuzzy +#| msgid "distribution description" +msgid "featureinfo type" +msgstr "distribution description" + +msgid "" +"specifies if the attribute should be rendered with an HTML widget on " +"GetFeatureInfo template." +msgstr "" + +msgid "count" +msgstr "" + +msgid "count value for this field" +msgstr "" + +msgid "min" +msgstr "" + +msgid "minimum value for this field" +msgstr "" + +msgid "max" +msgstr "" + +msgid "maximum value for this field" +msgstr "" + +msgid "average" +msgstr "" + +msgid "average value for this field" +msgstr "" + +msgid "median" +msgstr "" + +msgid "median value for this field" +msgstr "" + +msgid "standard deviation" +msgstr "" + +msgid "standard deviation for this field" +msgstr "" + +msgid "sum" +msgstr "" + +msgid "sum value for this field" +msgstr "" + +msgid "unique values for this field" +msgstr "" + +msgid "last modified" +msgstr "" + +msgid "date when attribute statistics were last updated" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Append to Dataset" +msgstr "Create a map" + +msgid "Return to Dataset" +msgstr "" + +msgid "Preserve Metadata XML" +msgstr "" + +msgid "Select the charset or leave default" +msgstr "" + +msgid "Home" +msgstr "" + +msgid "Change point of contact" +msgstr "" + +msgid "Assign a new point of contact to the layers below:" +msgstr "" + +msgid "Dataset WMS GetCapabilities document" +msgstr "" + +msgid "Filter Granules" +msgstr "" + +msgid "Active Filter:" +msgstr "" + +msgid "Granule ID" +msgstr "" + +msgid "Bounding Box" +msgstr "" + +msgid "Time" +msgstr "" + +msgid "Actions" +msgstr "" + +msgid "Granule Preview" +msgstr "" + +msgid "Granule Remove" +msgstr "" + +msgid "Dimension" +msgstr "" + +msgid "Enabled" +msgstr "" + +msgid "Regex" +msgstr "" + +msgid "TIME" +msgstr "" + +msgid "ELEVATION" +msgstr "" + +msgid "Attribute Name" +msgstr "" + +msgid "Range" +msgstr "" + +msgid "Average" +msgstr "" + +msgid "Median" +msgstr "" + +msgid "Standard Deviation" +msgstr "" + +msgid "Rate this dataset" +msgstr "" + +msgid "Analyze with" +msgstr "" + +msgid "Download Dataset" +msgstr "" + +msgid "Images" +msgstr "" + +msgid "Data" +msgstr "" + +msgid "Click to filter the layer" +msgstr "" + +msgid "Do you want to filter it?" +msgstr "" + +msgid "Loading..." +msgstr "" + +msgid "Filter by attributes" +msgstr "" + +msgid "Match" +msgstr "" + +msgid "all" +msgstr "" + +msgid "any" +msgstr "" + +msgid "of the following:" +msgstr "" + +msgid "Attribute" +msgstr "" + +msgid "Operator" +msgstr "" + +msgid "Feature limit" +msgstr "" + +msgid "Pick your download format:" +msgstr "" + +msgid "" +"No data available for this resource. Please contact a system administrator " +"or a manager." +msgstr "" + +msgid "Warning" +msgstr "" + +msgid "Edit Layer" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload Metadata" +msgstr "Edit Map Metadata" + +msgid "Styles" +msgstr "" + +msgid "Manage" +msgstr "" + +msgid "Layer" +msgid_plural "Layers" +msgstr[0] "" +msgstr[1] "" + +msgid "Append Data" +msgstr "" + +msgid "Edit data" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "View Dataset" +msgstr "Edit Map Metadata" + +msgid "Attribute Information" +msgstr "" + +msgid "ISO Feature Catalogue" +msgstr "" + +msgid "Legend" +msgstr "" + +#, fuzzy +#| msgid "Replace this Document" +msgid "Maps using this dataset" +msgstr "Replace this Document" + +msgid "List of maps using this dataset:" +msgstr "" + +msgid "This dataset is not currently used in any maps." +msgstr "" + +msgid "Create a map using this dataset" +msgstr "" + +msgid "Click the button below to generate a new map based on this dataset." +msgstr "" + +msgid "Add the dataset to an existing map" +msgstr "" + +msgid "Click the button below to add the dataset to the selected map." +msgstr "" + +#, fuzzy +#| msgid "Add to Favorites" +msgid "Add to Map" +msgstr "Add to Favorites" + +msgid "Documents related to this layer" +msgstr "" + +msgid "List of documents related to this layer:" +msgstr "" + +msgid "" +"The following styles are associated with this layer. Choose a style to view " +"it in the preview map." +msgstr "" + +msgid "(default style)" +msgstr "" + +msgid "No styles associated with this layer" +msgstr "" + +msgid "External service layer" +msgstr "" + +msgid "Source" +msgstr "" + +msgid "Refresh Attributes and Statistics of this layer" +msgstr "" + +msgid "" +"Click the button below to allow GeoNode refreshing the list of available " +"Dataset Attributes. If the option 'WPS_ENABLED' has been also set on the " +"backend, it will recalculate their statistics too." +msgstr "" + +msgid "Refresh Attributes and Statistics" +msgstr "" + +msgid "Clear the Server Cache of this layer" +msgstr "" + +msgid "Click the button below to wipe the tile-cache of this layer." +msgstr "" + +msgid "Empty Tiled-Dataset Cache" +msgstr "" + +msgid "Click the button below to change the permissions of this layer." +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "Change Dataset Permissions" +msgstr "You do not have permission to view this map" + +msgid "Remove Mosaic Granules" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove Granule %(granule_id)s of the " +"Mosaic %(layer_title)s?\n" +" " +msgstr "" + +msgid "This action affects the following maps:" +msgstr "" + +msgid "No maps are using this layer" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload Datasets" +msgstr "Edit Map Metadata" + +#, python-format +msgid "for %(layer_title)s" +msgstr "" + +msgid "" +"Note: this layer's orginal metadata was populated and preserved by importing " +"a metadata XML file.\n" +" This metadata cannot be edited." +msgstr "" + +msgid "" +"Note: this layer's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and " +"Dublin Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" + +#, python-format +msgid "" +"\n" +" Editing details for %(layer_title)s\n" +" " +msgstr "" + +msgid "Use a custom template?" +msgstr "" + +msgid "Display Order" +msgstr "" + +msgid "Display Type" +msgstr "" + +msgid "Visible" +msgstr "" + +msgid "ERROR" +msgstr "" + +msgid "Following items are required:" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload Dataset Metadata" +msgstr "Edit Map Metadata" + +msgid "(XML - ISO, FGDC, ebRIM, Dublin Core)" +msgstr "" + +msgid "Remove Datasets" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(layer_title)s?\n" +" " +msgstr "" + +msgid "Replace Dataset" +msgstr "" + +msgid "Manage Styles" +msgstr "" + +#, python-format +msgid "" +"\n" +" Manage Available Styles for " +"%(layer_title)s\n" +" " +msgstr "" + +msgid "Dataset Default Style" +msgstr "" + +msgid "Available styles" +msgstr "" + +msgid "" +"Click on an available style in the upper box to assign it to this layer. " +"Selected styles appear in the lower box." +msgstr "" + +msgid "Update Available Styles" +msgstr "" + +msgid "Return to Layer" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload Dataset Style" +msgstr "Edit Map Metadata" + +msgid "(SLD - Style Layer Descriptor 1.0, 1.1)" +msgstr "" + +msgid "WARNING" +msgstr "" + +msgid "This will most probably overwrite the current default style!" +msgstr "" + +msgid "Preview" +msgstr "" + +msgid "Dataset Attributes" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload Dataset" +msgstr "Edit Map Metadata" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload status" +msgstr "Edit Map Metadata" + +msgid "" +"This file needs additional configuration to complete the upload process. " +"Please click on the button to fill the required configuration" +msgstr "" + +msgid "Delete" +msgstr "" + +#, fuzzy +#| msgid "Add a Comment" +msgid "Upload process completed" +msgstr "Add a Comment" + +msgid "" +"The upload process cannot be completed because the original file is no more " +"available" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Created" +msgstr "Edit Map Metadata" + +msgid "Progress" +msgstr "" + +msgid "page" +msgstr "" + +msgid "of" +msgstr "" + +msgid "Delete incomplete upload" +msgstr "" + +msgid "Are you sure you want to remove this incomplete upload?" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload Dataset Step: Set SRS" +msgstr "Edit Map Metadata" + +msgid "Provide CRS for " +msgstr "" + +msgid "Coordinate Reference System" +msgstr "" + +msgid "" +"\n" +" A coordinate reference system for this dataset could not be " +"determined.\n" +" Locate or enter the appropriate ESPG code for this dataset " +"below.\n" +" One way to do this is do visit:\n" +" prj2epsg\n" +" and enter the following:\n" +" " +msgstr "" + +msgid "Native CRS could not be found!" +msgstr "" + +msgid "Provide EPSG code for Source SRS" +msgstr "" + +msgid "EPSG Code (Source SRS)" +msgstr "" + +msgid "Select a Source SRS" +msgstr "" + +msgid "" +"Source SRS EPSG Code is mandatory and represents the native data Spatial " +"Reference System.\n" +"

          \n" +" This must be coherent with the Geometry values (lon/lat " +"coordinates as an instance) stored on the geospatial dataset.\n" +"

          \n" +" If not specified on the geospatial data itself, it must " +"be manually declared by the operator.\n" +"

          \n" +" More information is provided at the bottom of the page " +"in the \"Additional Help\" sections.\n" +" " +msgstr "" + +msgid "Next" +msgstr "" + +msgid "Advanced Options" +msgstr "" + +msgid "Target SRS" +msgstr "" + +msgid "EPSG Code (Target SRS)" +msgstr "" + +msgid "Select a Target SRS" +msgstr "" + +msgid "leave it empty to use only Source SRS" +msgstr "" + +msgid "Additional Help" +msgstr "" + +msgid "Spatial Reference System" +msgstr "" + +msgid "" +"A spatial reference system (SRS) or coordinate reference system (CRS) is a " +"coordinate-based local,\n" +" regional or global system used to locate " +"geographical entities. A spatial reference system defines a specific map\n" +" projection, as well as transformations between " +"different spatial reference systems. Spatial reference systems are\n" +" defined by the OGC's Simple feature access using " +"well-known text, and support has been implemented by several\n" +" standards-based geographic information systems. " +"Spatial reference systems can be referred to using a SRID integer,\n" +" including EPSG codes defined by the " +"International Association of Oil and Gas Producers.\n" +" It is specified in ISO 19111:2007 Geographic " +"information—Spatial referencing by coordinates, also published as\n" +" OGC Abstract Specification, Topic 2: Spatial " +"referencing by coordinate." +msgstr "" + +msgid "Identifiers" +msgstr "" + +msgid "" +"\n" +" A Spatial Reference System Identifier (SRID) is a " +"unique value used to unambiguously identify projected, unprojected,\n" +" and local spatial coordinate system definitions. " +"These coordinate systems form the heart of all GIS applications.\n" +"\n" +" Virtually all major spatial vendors have created " +"their own SRID implementation or refer to those of an authority,\n" +" such as the European Petroleum Survey Group (EPSG).\n" +" " +msgstr "" + +msgid "" +"NOTE: As of 2005 the EPSG SRID values are now maintained by the " +"International\n" +" Association of Oil & Gas Producers (OGP) " +"Surveying & Positioning Committee" +msgstr "" + +msgid "" +"\n" +" SRIDs are the primary key for the Open Geospatial " +"Consortium (OGC) spatial_ref_sys metadata table for the Simple\n" +" Features for SQL Specification, Versions 1.1 and " +"1.2, which is defined as follows:\n" +" " +msgstr "" + +msgid "" +"\n" +" In spatially enabled databases (such as IBM DB2, IBM " +"Informix, Microsoft SQL Server, MySQL, Oracle RDBMS, Teradata, PostGIS and\n" +" SQL Anywhere), SRIDs are used to uniquely identify " +"the coordinate systems used to define columns of spatial data or individual\n" +" spatial objects in a spatial column (depending on " +"the spatial implementation). SRIDs are typically associated with a well " +"known\n" +" text (WKT) string definition of the coordinate " +"system (SRTEXT, above). From the Well Known Text Wikipedia page\n" +" " +msgstr "" + +msgid "" +"“A WKT string for a spatial reference system describes the datum, geoid, " +"coordinate system,\n" +" and map projection of the spatial objects”." +msgstr "" + +msgid "" +"\n" +" Here are two common coordinate systems with their " +"EPSG SRID value followed by their well known text:\n" +" " +msgstr "" + +msgid "" +"\n" +" UTM, Zone 17N, NAD27 — SRID 2029\n" +" " +msgstr "" + +msgid "" +"\n" +" WGS84 — SRID 4326\n" +" " +msgstr "" + +msgid "" +"\n" +" SRID values associated with spatial data can be used " +"to constrain spatial operations — for instance, spatial operations cannot be " +"performed\n" +" between spatial objects with differing SRIDs in some " +"systems, or trigger coordinate system transformations between spatial " +"objects in others.\n" +" " +msgstr "" + +msgid "" +"Source SRS EPSG Code is mandatory and represents the native data Spatial " +"Reference System. This must be coherent with the\n" +" Geometry values (lon/lat coordinates as an " +"instance) stored on the geospatial dataset. If not specified on the " +"geospatial data itself, it\n" +" must be manually declared by the operator." +msgstr "" + +msgid "" +"Target SRS EPSG Code is optional. This must be used only if we need to re-" +"project the coordinates from Source SRS to another one.\n" +" " +msgstr "" + +msgid "Upload Dataset Step: CSV Field Mapping" +msgstr "" + +msgid "Geospatial Data" +msgstr "" + +msgid "" +"Please indicate which attributes contain the latitude and longitude " +"coordinates in the CSV data." +msgstr "" + +msgid "" +"With this data, GeoNode was able to guess which attributes contain the\n" +" latitude and longitude coordinates, but please confirm that " +"the correct\n" +" attributes are selected below." +msgstr "" + +msgid "Select an attribute" +msgstr "" + +msgid "" +"We did not detect columns that could be used for the latitude and " +"longitude.\n" +" Please verify that you have two columns in your csv file that can be " +"used for\n" +" the latitude and longitude." +msgstr "" + +msgid "Your upload is either complete or you haven't resumed an earlier one." +msgstr "" + +msgid "Return to" +msgstr "" + +msgid "Upload Form" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Upload Dataset Step: Time" +msgstr "Edit Map Metadata" + +msgid "Inspect data for " +msgstr "" + +msgid "Configure as Time-Series" +msgstr "" + +msgid "" +"Toggling this selector allows you to configure (or not) this data as a time " +"series; in this case you will also have to select an attribute\n" +" to drive the time dimension.\n" +"

          \n" +" If GeoNode is not able to parse any of the values for " +"the selected attribute red markers will appear to highlight the problems.\n" +"

          \n" +" More information is provided at the bottom of the page " +"in the \"Additional Help\" sections.\n" +" " +msgstr "" + +msgid "No" +msgstr "" + +msgid "Use an existing timestamp attribute in the data" +msgstr "" + +msgid "Yes: with an existing Time-Attribute" +msgstr "" + +msgid "" +"Yes: by converting data to a timestamp using standard date/time " +"representation" +msgstr "" + +msgid "Convert a number field into a year" +msgstr "" + +msgid "Yes: by converting a number as Year" +msgstr "" + +msgid "" +"Convert data to a timestamp using standard date/time representation or a " +"custom format" +msgstr "" + +msgid "Start Importer" +msgstr "" + +msgid "Date Format" +msgstr "" + +msgid "Text Attribute Date Format" +msgstr "" + +msgid "Best Guess" +msgstr "" + +msgid "Custom" +msgstr "" + +msgid "Optional End-Time attribute" +msgstr "" + +msgid "Existing Time Attribute" +msgstr "" + +msgid "Convert Text Attribute" +msgstr "" + +msgid "Convert Number (As Year)" +msgstr "" + +msgid "Time attribute Presentation" +msgstr "" + +msgid "List" +msgstr "" + +msgid "of all the distinct time values" +msgstr "" + +msgid "Intervals" +msgstr "" + +msgid "defined by the resolution" +msgstr "" + +msgid "Continuous Intervals" +msgstr "" + +msgid "" +"for data that is frequently updated, resolution describes the frequency of " +"updates" +msgstr "" + +msgid "Resolution of time attribute" +msgstr "" + +msgid "Enabling Time" +msgstr "" + +msgid "" +"A dataset can support one or two time attributes. If a single\n" +" attribute is used, the dataset is considered to " +"contain data that is valid at single points in time. If two\n" +" attributes are used, the second attribute represents " +"the end of a valid period hence the dataset is considered\n" +" to contain data that is valid at certain periods in " +"time." +msgstr "" + +msgid "Selecting an Attribute" +msgstr "" + +msgid "A time attribute can be" +msgstr "" + +msgid "An existing date" +msgstr "" + +msgid "Text that can be converted to a timestamp" +msgstr "" + +msgid "A number representing a year" +msgstr "" + +msgid "" +"\n" +" For text attributes, one can specify a custom format " +"(as part of the \"Advanced Options\") or use the 'best guess' approach which " +"will try to\n" +" automatically translate well-known recognized " +"patterns into valid times.\n" +" " +msgstr "" + +msgid "The 'best guess' will handle date and optional time variants of" +msgstr "" + +msgid "In terms of the formatting flags noted above, these are" +msgstr "" + +msgid "Modal Header" +msgstr "" + +msgid "Some text in the modal" +msgstr "" + +msgid " does NOT match any valid ISO-8601 Date-Time string!" +msgstr "" + +msgid "Valid ISO-8601 for" +msgstr "" + +msgid "matches a valid ISO-8601 Date-Time string!" +msgstr "" + +msgid "Not Valid ISO-8601 for" +msgstr "" + +msgid "does NOT match any valid ISO-8601 Date-Time string!" +msgstr "" + +#, fuzzy +#| msgid "Your selections" +msgid "Wrong Selection" +msgstr "Your selections" + +msgid "Please, select one Time Attribute to test!" +msgstr "" + +msgid "" +"Returning to the upload starting page in 5seconds " +msgstr "" + +msgid " Or just go " +msgstr "" + +msgid "now" +msgstr "" + +#, python-brace-format +msgid "You are attempting to {action_type} a vector dataset with a raster." +msgstr "" + +#, python-brace-format +msgid "You are attempting to {action_type} a raster dataset with a vector." +msgstr "" + +#, python-brace-format +msgid "" +"You are attempting to {action_type} a vector dataset with an unknown format." +msgstr "" + +#, python-brace-format +msgid "" +"Please ensure the name is consistent with the file you are trying to " +"{action_type}." +msgstr "" + +msgid "Local GeoNode dataset has no geometry type." +msgstr "" + +#, python-brace-format +msgid "" +"Please ensure there is at least one geometry " +"type that is consistent with the file you are " +"trying to {action_type}." +msgstr "" + +msgid "The selected Dataset does not exists in the catalog." +msgstr "" + +msgid "Please ensure that the dataset structure is consistent " +msgstr "" + +msgid "" +"Some error occurred while trying to access the uploaded schema: {str(e)}" +msgstr "" + +msgid "" +"There was an error while attempting to upload your data. Please try again, " +"or contact and administrator if the problem continues." +msgstr "" + +msgid "" +"Note: this dataset's orginal metadata was populated and preserved by " +"importing a metadata XML file. This metadata cannot be edited." +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not permitted to delete this dataset" +msgstr "You do not have permission to view this layer" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You do not have permissions for this dataset." +msgstr "You do not have permission to view this layer" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not permitted to modify this dataset" +msgstr "You do not have permission to view this layer" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not permitted to modify this dataset's metadata" +msgstr "You do not have permission to view this layer" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not permitted to view this dataset" +msgstr "You do not have permission to view this layer" + +msgid "" +"This dataset is a member of a layer group, you must remove the dataset from " +"the group before deleting." +msgstr "" + +#, fuzzy, python-format +#| msgid "Remote Thumbnail" +msgid "couldn't generate thumbnail: %s" +msgstr "Remote Thumbnail" + +msgid "Map Created" +msgstr "" + +msgid "A Map was created" +msgstr "" + +msgid "Map Updated" +msgstr "" + +msgid "A Map was updated" +msgstr "" + +msgid "Map Approved" +msgstr "" + +msgid "A Map was approved by a Manager" +msgstr "" + +msgid "Map Published" +msgstr "" + +msgid "A Map was published" +msgstr "" + +msgid "Map Deleted" +msgstr "" + +msgid "A Map was deleted" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Comment on Map" +msgstr "Create a map" + +msgid "A map was commented on" +msgstr "" + +msgid "Rating for Map" +msgstr "" + +msgid "A rating was given to a map" +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Featured Map URL" +msgstr "Create a map" + +msgid "stack order" +msgstr "" + +msgid "format" +msgstr "" + +msgid "name" +msgstr "" + +msgid "store" +msgstr "" + +msgid "opacity" +msgstr "" + +#, fuzzy +#| msgid "Edit Styles" +msgid "styles" +msgstr "Edit Styles" + +msgid "transparent" +msgstr "" + +msgid "fixed" +msgstr "" + +msgid "group" +msgstr "" + +msgid "visibility" +msgstr "" + +msgid "ows URL" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "layer params" +msgstr "Edit Map Metadata" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "source params" +msgstr "Edit Map Metadata" + +msgid "Map layers WMS GetCapabilities document" +msgstr "" + +msgid "Rate this Map" +msgstr "" + +msgid "Download Map" +msgstr "" + +msgid "Map" +msgid_plural "Maps" +msgstr[0] "" +msgstr[1] "" + +msgid "Map Layers" +msgstr "" + +msgid "This map uses the following layers:" +msgstr "" + +msgid "Documents related to this map" +msgstr "" + +msgid "List of documents related to this map:" +msgstr "" + +msgid "Change Permissions of this Map" +msgstr "" + +msgid "Copy this map" +msgstr "" + +msgid "Duplicate this map and modify it for your own purposes" +msgstr "" + +msgid "Create a New Map" +msgstr "" + +msgid "" +"\n" +" Here you can download all the layers of this map that\n" +" are hosted on this GeoNode.\n" +" " +msgstr "" + +msgid "" +"\n" +"
          Could not find downloadable layers " +"for this map. You can go back to \n" +" " +msgstr "" + +msgid "" +"\n" +" Additionally, the map contains these layers which will not be " +"downloaded\n" +" due to security restrictions:\n" +" " +msgstr "" + +msgid "" +"\n" +" Finally, the map contains these layers which will not be downloaded\n" +" because they are not available directly from this GeoNode:\n" +" " +msgstr "" + +msgid "Downloading" +msgstr "" + +msgid "Retry" +msgstr "" + +msgid "Start downloading this map" +msgstr "" + +msgid "Download Complete" +msgstr "" + +msgid "Download Failed" +msgstr "" + +msgid "Edit Map" +msgstr "" + +msgid "Explore Maps" +msgstr "" + +msgid "" +"Note: this map's orginal metadata was populated by importing a metadata XML " +"file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" + +msgid "Return to Map" +msgstr "" + +msgid "Remove Map" +msgstr "" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove %(map_title)s?\n" +" " +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You are not permitted to delete this map." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You do not have permissions for this map." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You must be logged in to save this map" +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You are not permitted to save or edit this map." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not allowed to modify this map's metadata." +msgstr "You do not have permission to view this layer" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You are not allowed to view this map." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "Thumbnail" +msgid "error saving thumbnail" +msgstr "Thumbnail" + +msgid "Message received" +msgstr "" + +msgid "New message received in one of your threads" +msgstr "" + +msgid "Monitoring alert" +msgstr "" + +msgid "Alert situation reported by monitoring" +msgstr "" + +msgid "GeoNode Monitoring on {base_ctx['host']} " +msgstr "" + +msgid "Enter valid email addresses." +msgstr "" + +msgid "Show list of services" +msgstr "" + +msgid "" +"Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, last sync will be used." +msgstr "" + +msgid "" +"Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, now will be used." +msgstr "" + +msgid "Force check" +msgstr "" + +msgid "Format of audit log (xml, json)" +msgstr "" + +msgid "" +"Should old data be preserved (default: no, data older than settings." +"MONITORING_DATA_TTL will be removed)" +msgstr "" + +msgid "Should stop on first error occured (default: no)" +msgstr "" + +msgid "Should process and send notifications as well (default: no)" +msgstr "" + +msgid "Collect data from this service only" +msgstr "" + +msgid "Show list of metrics" +msgstr "" + +msgid "Show list of labels for metric" +msgstr "" + +msgid "Show list of resources for metric" +msgstr "" + +msgid "Data aggregation interval in seconds (default: 60)" +msgstr "" + +msgid "Metric name" +msgstr "" + +msgid "Show data for specific resource in resource_type=resource_name format" +msgstr "" + +msgid "Show data for specific resource" +msgstr "" + +msgid "Show data for specific label" +msgstr "" + +#, python-brace-format +msgid "Write result to file, default GEOIP_PATH: {settings.GEOIP_PATH}" +msgstr "" + +msgid "" +"Fetch database from specific url. If nothing provided, default {} will be " +"used" +msgstr "" + +msgid "Overwrite file if exists" +msgstr "" + +msgid "GeoNode" +msgstr "" + +msgid "GeoServer" +msgstr "" + +msgid "Host (GeoServer)" +msgstr "" + +msgid "Host (GeoNode)" +msgstr "" + +msgid "No resource" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Resource base" +msgstr "Edit Map Metadata" + +#, fuzzy +#| msgid "Edit Styles" +msgid "Style" +msgstr "Edit Styles" + +msgid "Admin" +msgstr "" + +msgid "Other" +msgstr "" + +msgid "Rate" +msgstr "" + +msgid "Count" +msgstr "" + +msgid "Value" +msgstr "" + +msgid "Value numeric" +msgstr "" + +msgid "Bytes" +msgstr "" + +msgid "Kilobytes" +msgstr "" + +msgid "Megabytes" +msgstr "" + +msgid "Gigabytes" +msgstr "" + +msgid "Bytes per second" +msgstr "" + +msgid "Kilobytes per second" +msgstr "" + +msgid "Megabytes per second" +msgstr "" + +msgid "Gigabytes per second" +msgstr "" + +msgid "Seconds" +msgstr "" + +msgid "Percentage" +msgstr "" + +msgid "Not OWS" +msgstr "" + +msgid "Any OWS" +msgstr "" + +msgid "Change" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Change Metadata" +msgstr "Edit Map Metadata" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "View Metadata" +msgstr "Edit Map Metadata" + +msgid "Publish" +msgstr "" + +msgid "Geoserver event" +msgstr "" + +msgid "List of resources affected" +msgstr "" + +#, fuzzy +#| msgid "Replace this Document" +msgid "Response processing time in ms" +msgstr "Replace this Document" + +msgid "1 minute" +msgstr "" + +msgid "5 minutes" +msgstr "" + +msgid "10 minutes" +msgstr "" + +msgid "30 minutes" +msgstr "" + +msgid "1 hour" +msgstr "" + +msgid "Error" +msgstr "" + +msgid "Fatal" +msgstr "" + +msgid "Expected min/max values for user configuration" +msgstr "" + +msgid "Marker of last delivery" +msgstr "" + +msgid "Minimum time between subsequent notifications" +msgstr "" + +msgid "How severe would be error from this notification" +msgstr "" + +msgid "Is it active" +msgstr "" + +#, python-brace-format +msgid "{self.severity}: {self.name}" +msgstr "" + +msgid "Value must be above" +msgstr "" + +msgid "Value must be below" +msgstr "" + +msgid "Last update must not be older than" +msgstr "" + +msgid "Max timeout for given metric before error should be raised" +msgstr "" + +msgid "Monitoring & Analytics" +msgstr "" + +msgid "Only superusers can monitor performance" +msgstr "" + +msgid "Hi," +msgstr "" + +msgid "Monitoring on" +msgstr "" + +msgid "has detected following problems :" +msgstr "" + +msgid "spotted" +msgstr "" + +msgid "ago" +msgstr "" + +msgid "emerged" +msgstr "" + +msgid "User following you" +msgstr "" + +msgid "Another user has started following you" +msgstr "" + +msgid "User requested access" +msgstr "" + +msgid "A new user has requested access to the site" +msgstr "" + +msgid "Account activated" +msgstr "" + +msgid "This account is now active and can log in the site" +msgstr "" + +msgid "Personal info" +msgstr "" + +msgid "Important dates" +msgstr "" + +msgid "Extended profile" +msgstr "" + +msgid "Password changed successfully." +msgstr "" + +#, python-format +msgid "Change password: %s" +msgstr "" + +msgid "party who authored the resource" +msgstr "" + +msgid "" +"party who has processed the data in a manner such that the resource has been " +"modified" +msgstr "" + +msgid "party who published the resource" +msgstr "" + +msgid "" +"party that accepts accountability and responsibility for the data and " +"ensures appropriate care and maintenance of the resource" +msgstr "" + +msgid "" +"party who can be contacted for acquiring knowledge about or acquisition of " +"the resource" +msgstr "" + +msgid "party who distributes the resource" +msgstr "" + +msgid "party who uses the resource" +msgstr "" + +msgid "party that supplies the resource" +msgstr "" + +msgid "party who created the resource" +msgstr "" + +msgid "party that owns the resource" +msgstr "" + +msgid "key party responsible for gathering information and conducting research" +msgstr "" + +msgid "Email Address" +msgstr "" + +msgid "Organization Name" +msgstr "" + +msgid "name of the responsible organization" +msgstr "" + +msgid "Profile" +msgstr "" + +msgid "introduce yourself" +msgstr "" + +msgid "Position Name" +msgstr "" + +msgid "role or position of the responsible person" +msgstr "" + +msgid "Voice" +msgstr "" + +msgid "" +"telephone number by which individuals can speak to the responsible " +"organization or individual" +msgstr "" + +msgid "Facsimile" +msgstr "" + +msgid "" +"telephone number of a facsimile machine for the responsible organization or " +"individual" +msgstr "" + +msgid "Delivery Point" +msgstr "" + +msgid "" +"physical and email address at which the organization or individual may be " +"contacted" +msgstr "" + +msgid "City" +msgstr "" + +msgid "city of the location" +msgstr "" + +msgid "Administrative Area" +msgstr "" + +msgid "state, province of the location" +msgstr "" + +msgid "Postal Code" +msgstr "" + +msgid "ZIP or other postal code" +msgstr "" + +msgid "Country" +msgstr "" + +msgid "country of the physical address" +msgstr "" + +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated" +msgstr "" + +msgid "Timezone" +msgstr "" + +msgid "No Group" +msgstr "" + +msgid "All contents" +msgstr "" + +msgid "No Organization Info" +msgstr "" + +msgid "Forgot Username" +msgstr "" + +msgid "" +"Enter your email address and click the submit button.
          Your username " +"will be sent to you." +msgstr "" + +msgid "Create Profile" +msgstr "" + +msgid "Profile of " +msgstr "" + +msgid "Not provided." +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Organization" +msgstr "" + +msgid "Not provided" +msgstr "" + +msgid "Fax" +msgstr "" + +msgid "User layers WMS GetCapabilities document" +msgstr "" + +msgid "Message User" +msgstr "" + +msgid "Edit profile" +msgstr "" + +msgid "Connected social accounts" +msgstr "" + +msgid "Associated e-mails" +msgstr "" + +msgid "Set/Change password" +msgstr "" + +msgid "Upload new layers" +msgstr "" + +msgid "Create a new layer" +msgstr "" + +#, fuzzy +#| msgid "Add a Comment" +msgid "Upload new document" +msgstr "Add a Comment" + +msgid "Create a new map" +msgstr "" + +msgid "My Activities" +msgstr "" + +msgid "Notifications" +msgstr "" + +msgid "Announcements" +msgstr "" + +msgid "Invite Users" +msgstr "" + +msgid "User Activities" +msgstr "" + +msgid "Resources" +msgstr "" + +msgid "Edit Your Profile" +msgstr "" + +msgid "Edit Profile for" +msgstr "" + +msgid "Change your avatar" +msgstr "" + +msgid "Update profile" +msgstr "" + +msgid "Explore People" +msgstr "" + +#, python-brace-format +msgid "Your username for {site.name}" +msgstr "" + +msgid "Your username has been emailed to you." +msgstr "" + +msgid "No user could be found with that email address." +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You are not permitted to save or edit this resource." +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "You are not authorized to download this resource." +msgstr "You do not have permission to view this document" + +msgid "" +"No files have been found for this resource. Please, contact a system " +"administrator." +msgstr "" + +msgid "No files found." +msgstr "" + +msgid "Not Authorized" +msgstr "" + +msgid "GeoNode Resource Processing Library" +msgstr "" + +msgid "Disabling this Task will make the Processing Workflow to skip it." +msgstr "" + +msgid "Processing Workflow Tasks" +msgstr "" + +msgid "Session is Expired. Please login again!" +msgstr "" + +msgid "Permissions successfully updated!" +msgstr "" + +msgid "Error updating permissions :(" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not allowed to change permissions for this resource" +msgstr "You do not have permission to view this layer" + +msgid "Security Rules Cache Refreshed!" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You cannot modify this resource!" +msgstr "You do not have permission to view this map" + +msgid "Attributes/Stats Refreshed Successfully!" +msgstr "" + +msgid "GeoWebCache Tiled Layer Emptied!" +msgstr "" + +msgid "Wrong permissions specification" +msgstr "" + +msgid "error delivering notification" +msgstr "" + +msgid "Auto-detect" +msgstr "" + +msgid "Paired WMS/WFS/WCS" +msgstr "" + +msgid "Web Map Service" +msgstr "" + +msgid "Catalogue Service" +msgstr "" + +msgid "ArcGIS REST MapServer" +msgstr "" + +msgid "ArcGIS REST ImageServer" +msgstr "" + +msgid "OpenGeoPortal" +msgstr "" + +msgid "Harvard Geospatial Library" +msgstr "" + +msgid "GeoNode (Web Map Service)" +msgstr "" + +msgid "GeoNode (Catalogue Service)" +msgstr "" + +msgid "Service URL" +msgstr "" + +msgid "Service Type" +msgstr "" + +#, python-format +msgid "Service %(url)s is already registered" +msgstr "" + +#, python-format +msgid "Could not connect to the service at %(url)s" +msgstr "" + +#, python-format +msgid "Could not find importable resources for the service at %(url)s" +msgstr "" + +#, python-format +msgid "Found service of type %(found_type)s instead of %(service_type)s" +msgstr "" + +msgid "Local" +msgstr "" + +msgid "Cascaded" +msgstr "" + +msgid "Harvested" +msgstr "" + +msgid "Indexed" +msgstr "" + +msgid "Live" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Resource is queued" +msgstr "Edit Map Metadata" + +msgid "Remote service unavailable" +msgstr "" + +msgid "Service unavailable" +msgstr "" + +msgid "Could not contact" +msgstr "" + +msgid "Contact" +msgstr "" + +msgid "Service Resources" +msgstr "" + +msgid "No resources have been imported yet." +msgstr "" + +msgid "Retry job" +msgstr "" + +msgid "previous" +msgstr "" + +msgid "next" +msgstr "" + +msgid "Connecting to service..." +msgstr "" + +msgid "Edit Service Metadata" +msgstr "" + +msgid "Import Service Resources" +msgstr "" + +msgid "Remove Service" +msgstr "" + +msgid "Edit Service" +msgstr "" + +msgid "Services" +msgstr "" + +msgid "Add Remote Service" +msgstr "" + +msgid "Remote Services" +msgstr "" + +msgid "No services registered" +msgstr "" + +msgid "Register Service" +msgstr "" + +msgid "Register New Service" +msgstr "" + +msgid "Service has been created:" +msgstr "" + +msgid "The following layers will be imported" +msgstr "" + +msgid "Remove Remote Service" +msgstr "" + +msgid "Are you sure you want to remove" +msgstr "" + +msgid "Deleting service and its associated resources..." +msgstr "" + +msgid "Import resources" +msgstr "" + +msgid "no" +msgstr "" + +msgid "resources can be imported" +msgstr "" + +msgid "- These will be cascaded through your local geoserver instance" +msgstr "" + +msgid "Id" +msgstr "" + +#, fuzzy +#| msgid "Clear all filters" +msgid "Clear Filter" +msgstr "Clear all filters" + +msgid "Back to service details" +msgstr "" + +msgid "Import Resources" +msgstr "" + +msgid "Previous" +msgstr "" + +msgid "All resources have already been imported" +msgstr "" + +msgid "Harvesting resources..." +msgstr "" + +msgid "Re-scan Service for new Resources" +msgstr "" + +msgid "Service registered successfully" +msgstr "" + +msgid "The selected resources are being imported" +msgstr "" + +msgid "The selected resources have been imported" +msgstr "" + +msgid "Resource is already being processed" +msgstr "" + +#, fuzzy, python-brace-format +#| msgid "Edit Map Metadata" +msgid "Resource {resource_id} is being processed" +msgstr "Edit Map Metadata" + +msgid "Service rescanned successfully" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this map" +msgid "You dont have enougth rigths to see the resource detail" +msgstr "You do not have permission to view this map" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not permitted to change this service." +msgstr "You do not have permission to view this layer" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "You are not permitted to remove this service." +msgstr "You do not have permission to view this layer" + +#, python-brace-format +msgid "Service {service.name} has been deleted" +msgstr "" + +msgid "created" +msgstr "" + +msgid "deleted" +msgstr "" + +msgid "updated" +msgstr "" + +msgid "added a comment" +msgstr "" + +msgid "updated a comment" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "uploaded" +msgstr "Edit Map Metadata" + +msgid "Recent activity" +msgstr "" + +msgid "to" +msgstr "" + +msgid "on" +msgstr "" + +msgid "Not Permitted" +msgstr "" + +msgid "" +"\n" +" You are not allowed to perform this operation.\n" +" " +msgstr "" + +msgid "Please verify that you are logged in as the correct user." +msgstr "" + +msgid "Please login or register and retry." +msgstr "" + +msgid "Page Not Found" +msgstr "" + +msgid "" +"\n" +" The page you requested does not exist. Perhaps you are using an " +"outdated bookmark?\n" +" " +msgstr "" + +msgid "Toggle navigation" +msgstr "" + +msgid "Datasets" +msgstr "" + +msgid "People" +msgstr "" + +msgid "Search" +msgstr "" + +msgid "Register" +msgstr "" + +msgid "Sign in" +msgstr "" + +msgid "You are using an outdated browser that is not supported by GeoNode." +msgstr "" + +msgid "" +"Please use a modern browser like Mozilla Firefox, Google " +"Chrome or Safari." +msgstr "" + +msgid "There was a problem loading this page" +msgstr "" + +msgid "" +"\n" +" Please contact your GeoNode administrator (they may have " +"received an email automatically if they configured it properly).\n" +" If you are the site administrator, enable debug mode to see the " +"actual error and fix it or file an issue in GeoNode's issue tracker\n" +" " +msgstr "" + +#, fuzzy +#| msgid "Create a map" +msgid "Create a new dataset" +msgstr "Create a map" + +#, fuzzy +#| msgid "Create a map" +msgid "Create Map" +msgstr "Create a map" + +msgid "Add User" +msgstr "" + +msgid "Powered by" +msgstr "" + +msgid "version" +msgstr "" + +msgid "Developers" +msgstr "" + +msgid "Username" +msgstr "" + +msgid "Password" +msgstr "" + +msgid "Remember Me" +msgstr "" + +msgid "Menu" +msgstr "" + +msgid "Recent Activity" +msgstr "" + +msgid "Inbox" +msgstr "" + +msgid "Help" +msgstr "" + +msgid "Log out" +msgstr "" + +msgid "Info" +msgstr "" + +msgid "Granules" +msgstr "" + +msgid "Share" +msgstr "" + +msgid "Ratings" +msgstr "" + +msgid "Exif" +msgstr "" + +msgid "Set permissions on selected resources" +msgstr "" + +msgid "Apply Changes" +msgstr "" + +msgid "total" +msgstr "" + +msgid "Add Comment" +msgstr "" + +msgid "Log in to add a comment" +msgstr "" + +msgid "By" +msgstr "" + +msgid "Submit Comment" +msgstr "" + +msgid "Who can view it?" +msgstr "" + +msgid "Anyone" +msgstr "" + +msgid "The following users:" +msgstr "" + +msgid "The following groups:" +msgstr "" + +msgid "Who can download it?" +msgstr "" + +msgid "Who can change metadata for it?" +msgstr "" + +msgid "Who can edit data for this dataset?" +msgstr "" + +msgid "Who can edit styles for this dataset?" +msgstr "" + +msgid "" +"Who can manage it? (update, delete, change permissions, publish/unpublish it)" +msgstr "" + +msgid "Set permissions for this resource" +msgstr "" + +msgid "People and Groups" +msgstr "" + +msgid "Geo Limits" +msgstr "" + +msgid "Users" +msgstr "" + +msgid "Show Geo Limits" +msgstr "" + +#, fuzzy +#| msgid "Add a Comment" +msgid "Upload Geo Limits" +msgstr "Add a Comment" + +msgid "Save Geo Limits" +msgstr "" + +msgid "Delete Geo Limits" +msgstr "" + +msgid "Message box" +msgstr "" + +msgid "OK" +msgstr "" + +msgid "Mandatory files : SHP , DBF" +msgstr "" + +msgid "" +"Upload a ZIP file containing an ESRI Shapefile. If the ZIP provides also a ." +"prj file, you don't have to specify the EPSG SRID" +msgstr "" + +msgid "Choose" +msgstr "" + +msgid "Choose a file..." +msgstr "" + +msgid "Reset" +msgstr "" + +msgid "Only the SRID number without the 'EPSG:' prefix!" +msgstr "" + +msgid "Default : 4326" +msgstr "" + +msgid "Set your desired encoding UTF-8, Big5, Big5-HKSCS ..." +msgstr "" + +msgid "Load SHP-ZIP" +msgstr "" + +msgid "" +"

          This will remove the Geo Limits currently drawn on the map.

          In " +"order to store the Geo Limits you will need to save them " +"anyway.

          Do you want to proceed?

          " +msgstr "" + +msgid "" +"

          This will override the current stored Geo Limits on the DB.

          To " +"apply them you will need to click on Apply Changes button " +"anyway.

          WARNING: This operation cannot be reverted!

          Do you want to proceed?

          " +msgstr "" + +msgid "Save Geo Limit" +msgstr "" + +msgid "" +"

          Geometry successfully saved!

          To apply them you " +"will need to click on Apply Changes button.

          " +msgstr "" + +msgid "

          Error while trying to save the Geometry!

          " +msgstr "" + +msgid "Load Geo Limits" +msgstr "" + +msgid "Load Geo Limit" +msgstr "" + +msgid "

          No Geometry found!

          " +msgstr "" + +msgid "" +"

          This will permanently remove the Geo Limits from the DB." +"

          To apply them you will need to click on Apply Changes button.

          Do you want to proceed?

          " +msgstr "" + +msgid "Delete Geo Limit" +msgstr "" + +msgid "

          Geo Limitsy successfully deleted!

          " +msgstr "" + +msgid "" +"

          Error occurred while trying to delete Geo Limits:

          " +msgstr "" + +msgid "Choose users..." +msgstr "" + +msgid "Choose groups..." +msgstr "" + +msgid "Placeholder for status-message" +msgstr "" + +msgid "Placeholder for status-message-body" +msgstr "" + +msgid "About GeoNode" +msgstr "" + +msgid "" +"GeoNode is a geospatial content management system, a platform for the " +"management and publication of geospatial data. It brings together mature and " +"stable open-source software projects under a consistent and easy-to-use " +"interface allowing non-specialized users to share data and create " +"interactive maps." +msgstr "" + +msgid "" +"Data management tools built into GeoNode allow for integrated creation of " +"data, metadata, and map visualizations. Each dataset in the system can be " +"shared publicly or restricted to allow access to only specific users. Social " +"features like user profiles and commenting and rating systems allow for the " +"development of communities around each platform to facilitate the use, " +"management, and quality control of the data the GeoNode instance contains." +msgstr "" + +msgid "" +"It is also designed to be a flexible platform that software developers can " +"extend, modify or integrate against to meet requirements in their own " +"applications." +msgstr "" + +msgid "Account Inactive" +msgstr "" + +msgid "This account is inactive." +msgstr "" + +msgid "Account Pending Approval" +msgstr "" + +#, python-format +msgid "" +"We have sent the administrators a notice to approve your account associated " +"with %(email)s. If the account is approved, you will receive a " +"confirmation notice." +msgstr "" + +msgid "Account" +msgstr "" + +msgid "E-mail Addresses" +msgstr "" + +msgid "The following e-mail addresses are associated with your account:" +msgstr "" + +msgid "Verified" +msgstr "" + +msgid "Unverified" +msgstr "" + +#, fuzzy +#| msgid "Primary email" +msgid "Primary" +msgstr "Primary email" + +msgid "Make Primary" +msgstr "" + +msgid "Re-send Verification" +msgstr "" + +msgid "Warning:" +msgstr "" + +msgid "" +"You currently do not have any e-mail address set up. You should really add " +"an e-mail address so you can receive notifications, reset your password, etc." +msgstr "" + +msgid "Add E-mail Address" +msgstr "" + +msgid "Add E-mail" +msgstr "" + +msgid "Do you really want to remove the selected e-mail address?" +msgstr "" + +#, python-format +msgid "" +"Hello from %(site_name)s!\n" +"\n" +"You're receiving this e-mail because user %(user_display)s has given yours " +"as an e-mail address to connect their account.\n" +"\n" +"To confirm this is correct, go to %(activate_url)s\n" +msgstr "" + +#, python-format +msgid "" +"Thank you from %(site_name)s!\n" +"%(site_domain)s" +msgstr "" + +#, python-format +msgid "Please confirm email address for %(site_name)s" +msgstr "" + +#, python-format +msgid "You have been invited to sign up at %(site_name)s." +msgstr "" + +#, python-format +msgid "Create an account on %(site_name)s" +msgstr "" + +msgid "" +"This is the email notification to confirm your password has been changed on" +msgstr "" + +msgid "Change password email notification" +msgstr "" + +#, python-format +msgid "" +"Hello from %(site_name)s!\n" +"\n" +"You're receiving this e-mail because you or someone else has requested a " +"password for your user account.\n" +"It can be safely ignored if you did not request a password reset. Click the " +"link below to reset your password." +msgstr "" + +#, python-format +msgid "In case you forgot, your username is %(username)s." +msgstr "" + +#, python-format +msgid "" +"Thank you for using %(site_name)s!\n" +"%(site_domain)s" +msgstr "" + +#, python-format +msgid "[%(site_name)s] Password reset" +msgstr "" + +msgid "Confirm E-mail Address" +msgstr "" + +#, python-format +msgid "" +"Please confirm that %(email)s is an e-mail " +"address for user %(user_display)s." +msgstr "" + +msgid "Confirm" +msgstr "" + +#, python-format +msgid "" +"This e-mail confirmation link expired or is invalid. Please issue a new e-mail confirmation request." +msgstr "" + +msgid "Log in" +msgstr "" + +msgid "Log in to an existing account" +msgstr "" + +#, python-format +msgid "" +"Please sign in with one\n" +" of your existing third party accounts. Or, sign up\n" +" for a %(site_name)s account and sign in below:" +msgstr "" + +msgid "or" +msgstr "" + +#, python-format +msgid "" +"If you have not created an account yet, then please\n" +" sign up first." +msgstr "" + +msgid "Forgot Password?" +msgstr "" + +msgid "Sign In" +msgstr "" + +msgid "Are you sure you want to log out?" +msgstr "" + +msgid "Password change" +msgstr "" + +msgid "Password Change" +msgstr "" + +msgid "Change your password here" +msgstr "" + +msgid "Change my password" +msgstr "" + +msgid "Password reset" +msgstr "" + +msgid "Password Reset" +msgstr "" + +msgid "Forgotten your password?" +msgstr "" + +msgid "" +"Enter your email address below, and we'll send you an email allowing you to " +"reset it." +msgstr "" + +msgid "Reset my password" +msgstr "" + +#, python-format +msgid "" +"If you have any trouble resetting your password, contact us at %(THEME_ACCOUNT_CONTACT_EMAIL)s." +msgstr "" + +msgid "" +"We have sent you an e-mail. Please contact us if you do not receive it " +"within a few minutes." +msgstr "" + +msgid "Change Password" +msgstr "" + +msgid "Bad Token" +msgstr "" + +#, python-format +msgid "" +"The password reset link was invalid, possibly because it has already been " +"used. Please request a new password reset." +msgstr "" + +msgid "change password" +msgstr "" + +msgid "Your password is now changed." +msgstr "" + +msgid "Set Password" +msgstr "" + +msgid "Password Set" +msgstr "" + +msgid "Set your password here" +msgstr "" + +msgid "Set password" +msgstr "" + +msgid "Sign up" +msgstr "" + +msgid "" +"Sign up with one\n" +" of your existing third party accounts" +msgstr "" + +msgid "Create a new local account" +msgstr "" + +msgid "Sign Up Closed" +msgstr "" + +msgid "We are sorry, but the sign up is currently closed." +msgstr "" + +msgid "Note" +msgstr "Note" + +#, python-format +msgid "you are already logged in as %(user_display)s." +msgstr "" + +msgid "Verify your email address" +msgstr "" + +msgid "" +"We have sent an email to you for verification. Follow the\n" +" link provided to finalize the signup process. Please contact us if you " +"do\n" +" not receive it within a few minutes" +msgstr "" + +msgid "Verify Your E-mail Address" +msgstr "" + +msgid "" +"This part of the site requires us to verify that\n" +"you are who you claim to be. For this purpose, we require that you\n" +"verify ownership of your e-mail address. " +msgstr "" + +msgid "" +"We have sent an e-mail to you for\n" +"verification. Please click on the link inside this e-mail. Please\n" +"contact us if you do not receive it within a few minutes." +msgstr "" + +#, python-format +msgid "" +"Note: you can still change your e-" +"mail address." +msgstr "" + +msgid "My Activity feed" +msgstr "" + +msgid "Activity Feed" +msgstr "" + +msgid "Are you sure you want to run the selected" +msgstr "" + +msgid "Yes, I'm sure" +msgstr "" + +msgid "GeoNode site admin" +msgstr "" + +msgid "GeoNode administration" +msgstr "" + +msgid "Delete Announcement?" +msgstr "" + +msgid "Are you sure you want to delete this announcement?" +msgstr "" + +msgid "Edit Announcement" +msgstr "" + +msgid "Create Announcement" +msgstr "" + +msgid "New Announcement" +msgstr "" + +msgid "Level" +msgstr "" + +msgid "Announcement" +msgstr "" + +msgid "Published From" +msgstr "" + +#, python-format +msgid "" +"\n" +" Published from %(publish_start)s to %(publish_end)s.\n" +" " +msgstr "" + +#, python-format +msgid "Announcements : %(announcement)s" +msgstr "" + +msgid "Back to edit your profile information" +msgstr "" + +msgid "Your current avatar: " +msgstr "" + +msgid "You haven't uploaded an avatar yet. Please upload one now." +msgstr "" + +msgid "Upload New Image" +msgstr "" + +msgid "Choose new Default" +msgstr "" + +msgid "Delete Your Avatar" +msgstr "" + +msgid "Please select the avatars that you would like to delete." +msgstr "" + +#, python-format +msgid "" +"You have no avatars to delete. Please upload one now." +msgstr "" + +msgid "Delete These" +msgstr "" + +msgid "GeoNode Search" +msgstr "" + +msgid "Contact Us" +msgstr "" + +msgid "Tel" +msgstr "" + +msgid "Uploading..." +msgstr "" + +msgid "Upload in progress..." +msgstr "" + +msgid "Processing..." +msgstr "" + +#, fuzzy +#| msgid "Thumbnail" +msgid "Updating Thumbnail..." +msgstr "Thumbnail" + +msgid "Message. Do you want to proceed?" +msgstr "" + +msgid "Processing Resource..." +msgstr "" + +msgid "Information for Developers" +msgstr "" + +msgid "Useful information for developers interested in GeoNode." +msgstr "" + +#, python-format +msgid "" +"\n" +"

          GeoNode is an open service built " +"on open source software. We encourage you to build new applications using " +"the components and resources it provides. This page is a starting point for " +"developers interesting in taking full advantage of GeoNode. It also includes " +"links to the project's source code so anyone can build and customize their " +"own GeoNode.

          \n" +"\n" +"

          GeoNode Software

          \n" +"\n" +"

          All the code that runs GeoNode is open source. The code is available " +"at http://github.com/GeoNode/" +"geonode/. The issue tracker for the project is at http://github.com/GeoNode/geonode/" +"issues.

          \n" +"\n" +"

          GeoNode is built using several open source projects, each with its " +"own community. If you are interested in contributing new features to the " +"GeoNode, we encourage you to do so by contributing to one of the projects on " +"which it is built:

          \n" +"
            \n" +"
          • GeoServer - Standards based " +"server for geospatial information
            • \n" +"
          • GeoWebCache - Cache engine " +"for WMS Tiles
          • OpenLayers - " +"Pure JavaScript library powering the maps of GeoExt
            • \n" +"
          • pycsw - CSW, OpenSearch and OAI-" +"PMH metadata catalogue server
            • \n" +"
          \n" +"\n" +"

          What are OGC Services?

          \n" +"

          The data in this application is served using open standards endorsed " +"by ISO and the Open Geospatial " +"Consortium; in particular, WMS (Web Map Service) is used for accessing " +"maps, WFS (Web Feature Service) is used for accessing vector data, and WCS " +"(Web Coverage Service) is used for accessing raster data. WMC (Web Map " +"Context Documents) is used for sharing maps. You can use these services in " +"your own applications using libraries such as OpenLayers, GeoTools, and OGR " +"(all of which are open-source software and available at zero cost). " +"Additionally, CSW (Catalog Service for the Web) supports access to " +"collections of descriptive information (metadata) about data and services.\n" +"\n" +"

          What is GeoWebCache?

          \n" +"

          GeoWebCache provides mapping tiles that are compatible with a number " +"of mapping engines, including Google Maps, Bing Maps and OpenLayers. All the " +"data hosted by GeoNode is also available through GeoWebCache. GeoWebCache " +"improves on WMS by caching data and providing more responsive maps.

          \n" +"\n" +"

          CSW Example Code

          \n" +"

          To interact with GeoNode's CSW you can use any CSW client (QGIS " +"MetaSearch, GRASS, etc.). The following example illustrates a simple " +"invocation using the OWSLib Python package:

          \n" +"

          from owslib.csw import CatalogueServiceWeb

          \n" +"

          from owslib.fes import PropertyIsLike

          \n" +"

          csw = CatalogueServiceWeb('%(CATALOGUE_BASE_URL)s')

          \n" +"

          anytext = PropertyIsLike('csw:AnyText', 'birds')')

          \n" +"

          csw.getrecords2(constraints=[anytext])

          \n" +"

          print csw.results

          \n" +"

          print csw.records

          \n" +"\n" +"

          OpenLayers Example Code

          \n" +"\n" +"

          To include a GeoNode map layer in an OpenLayers map, first find the " +"name for that layer. This is found in the layer's name field " +"(not title) of the layer list. For this example, we will use " +"the Nicaraguan political boundaries background layer, whose name is " +"risk:nicaragua_admin. Then, create an instance of OpenLayers." +"Layer.WMS:

          \n" +"

          var geonodeLayer = new OpenLayers.Layer.WMS(\"GeoNode Risk Data" +"\", \"http://demo.geonode.org/geoserver/wms\",{ layers: \"risk:" +"nicaragua_admin\" });

          \n" +"\n" +"

          Google Maps Example Code

          \n" +"

          To include a GeoNode map layer in a Google Map, include the dataset " +"name in the URL template.

          \n" +"

          var tilelayer = new GTileLayer(null, null, null, " +"{tileUrlTemplate: 'http://demo.geonode.org/geoserver/gwc/service/gmaps?" +"layers=risk:nicaragua_admin&zoom={Z}&x={X}&y={Y}', isPng:true, " +"opacity:0.5 } );

          \n" +"\n" +"

          Shapefile/GeoJSON/GML Output

          \n" +"

          To get data from the GeoNode web services use the WFS protocol. For " +"example, to get the full Nicaraguan admin boundaries use:

          \n" +"

          http://demo.geonode.org/geoserver/wfs?request=GetFeature&" +"typeName=risk:nicaragua_admin&outputformat=SHAPE-ZIP

          \n" +"

          Changing output format to json, GML2, " +"GML3, or csv will get data in those formats. The " +"WFS protocol also can handle more precise queries, specifying a bounding box " +"or various spatial and non-spatial filters based on the attributes of the " +"data.

          \n" +"\n" +"

          GeoTools Example Code

          \n" +"

          Create a DataStore and extract a FeatureType from it, then run a " +"Query. It is all documented on the wiki at http://geotools.org/." +"

          \n" +" " +msgstr "" + +msgid "GeoNode's Web Services" +msgstr "" + +msgid "GeoNode's Web Services are available from the following URLs:" +msgstr "" + +msgid "Dynamic tiles via WMS:" +msgstr "" + +msgid "Vector data via WFS:" +msgstr "" + +msgid "Raster data via WCS:" +msgstr "" + +msgid "Metadata search via CSW:" +msgstr "" + +msgid "Metadata search via OpenSearch:" +msgstr "" + +msgid "Metadata search via OAI-PMH:" +msgstr "" + +msgid "Cached tiles via WMTS:" +msgstr "" + +msgid "User Guide" +msgstr "" + +msgid "GeoNode Help" +msgstr "" + +msgid "" +"This page provides helpful information about how to use the GeoNode. You can " +"use the sidebar links to navigate to what you want to know." +msgstr "" + +msgid "" +"\n" +"

          The GeoNode provides access to data sets and a map editing " +"application allows users to browse existing maps and contribute their own.\n" +"\n" +"

          Browsing Layers

          \n" +"

          The Layers tab allows you to browse data " +"uploaded to this GeoNode.

          \n" +"

          All data can be downloaded in a variety of formats, for use in other " +"applications.

          \n" +"\n" +"

          Developer Access

          \n" +"

          The Developer page is the place for " +"developers to get started building applications against the GeoNode. It " +"includes instructions on using the web services, links to the source code of " +"the GeoNode, and information about the open source projects used to create " +"it.

          \n" +"\n" +"

          Browsing Maps

          \n" +"

          The GeoNode allows users to create and share maps with one another.\n" +"

          The Maps tab is a gateway to map exploration on " +"GeoNode. From here you can search for a map or " +"create a map, which will open the Map " +"Composer.

          \n" +"
          Google Earth Mode
          \n" +"

          Any map viewed in the interactive map editor can be seen in 3D mode " +"with the Google Earth plugin. To switch to 3D mode select the Google Earth " +"globe logo, the rightmost button on top toolbar. If you do not have the " +"Google Earth plugin installed you will be prompted to install it.

          \n" +"\n" +"

          Creating a Map

          \n" +"

          To create a new map go to the Contributed Maps " +"tab and click the create your own map link.

          \n" +"

          This will take you to the Map Composer with " +"a base layer loaded.

          \n" +"

          To add data layers from the GeoNode click on the green plus button " +"located below the layers tab on the left hand side of the screen. This will " +"open a dialog listing all the layers available on the GeoNode.

          \n" +"

          To add layers to your map select them and hit the Add Layers button. When finished you may hit Done to close the " +"dialog and go back to the map.

          \n" +"\n" +"
          Reordering and Removing Layers
          \n" +"

          Change the display order of the layers listed in the data tab by " +"simply dragging and dropping their names. The order in the map will be " +"updated to reflect that. To turn a layer's visibility off simply uncheck it, " +"and to remove it entirely select it and hit the red minus button.

          \n" +"\n" +"
          Saving your map
          \n" +"

          Once a suitable set of layers and zoom level has been found it's time " +"to save it so others can see it. Click the Save button--the left most icon " +"on the top toolbar, an image of a map with a disk--on the top menu and fill " +"out the title and abstract of the map.

          \n" +"\n" +"

          Publishing a Map

          \n" +"

          Any map from the GeoNode can be embedded for use in another site or " +"blog. To publish a map:

          \n" +"
            \n" +"
          1. Select the from the list of maps on the community or map page and " +"then hit the 'Publish map' button.
            2. \n" +"
          2. Choose your desired height and width for the widget in the wizard." +"
            3. \n" +"
          3. Copy the HTML snippet provided in the wizard to any HTML page or " +"iFrame-supporting blog post.
            4. \n" +"
          \n" +"

          This will put an interactive widget showing you map in your web page " +"or blog post.

          \n" +"

          Note that the Map Composer also has a " +"button to publish the map. Just be sure to save the map before publishing if " +"there are changes that you want others to see. It publishes the last saved " +"version, not the last viewed version.

          \n" +"\n" +"

          Remixing a Map

          \n" +"

          Any map available can serve as a starting point for a new map.

          \n" +"
            \n" +"
          1. Open the map in the Map Composer.
            2. \n" +"
          2. Add and remove layers as you like.
            3. \n" +"
          3. Pan and zoom to highlight the area of interest.
            4. \n" +"
          4. IMPORTANT: Update the title, abstract, contact " +"info and tags to reflect the new map.
            5. \n" +"
          5. Save the map.
            6. \n" +"
          \n" +"

          You will be able to see your new map when you search for it from the " +"Maps tab.

          \n" +" " +msgstr "" + +msgid "Sections" +msgstr "" + +msgid "Browsing Layers" +msgstr "" + +msgid "Developer Access" +msgstr "" + +msgid "Browsing Maps" +msgstr "" + +msgid "Creating a Map" +msgstr "" + +msgid "Exporting a Map" +msgstr "" + +msgid "Remixing a Map" +msgstr "" + +msgid "Welcome!" +msgstr "" + +msgid "Get Started »" +msgstr "" + +msgid "Search for Data." +msgstr "" + +msgid "Advanced Search" +msgstr "" + +msgid "" +"Click to search for geospatial data published by other users, organizations " +"and public sources. Download data in standard formats." +msgstr "" + +#, fuzzy +#| msgid "Add a Comment" +msgid "Add datasets" +msgstr "Add a Comment" + +msgid "Explore datasetss" +msgstr "" + +msgid "" +"Data is available for browsing, aggregating and styling to generate maps " +"which can be saved, downloaded, shared publicly or restricted to specify " +"users only." +msgstr "" + +msgid "Create maps" +msgstr "" + +msgid "Explore maps" +msgstr "" + +msgid "" +"As for the layers and maps GeoNode allows to publish tabular and text data, " +"manage their metadata and associated documents." +msgstr "" + +#, fuzzy +#| msgid "Add a Comment" +msgid "Add documents" +msgstr "Add a Comment" + +msgid "Explore documents" +msgstr "" + +msgid "" +"Geonode allows registered users to easily upload geospatial data and various " +"documents in several formats." +msgstr "" + +msgid "See users" +msgstr "" + +msgid "Featured Datasets" +msgstr "" + +msgid "Accept" +msgstr "" + +msgid "Reject" +msgstr "" + +msgid "Leave" +msgstr "" + +msgid "Dear Sir/Madam," +msgstr "" + +#, python-format +msgid "is inviting you to join (%(site_name)s)." +msgstr "" + +#, python-format +msgid "" +"To do so, please register at %(site_name)s " +"Registration." +msgstr "" + +msgid "" +"Once you receive the confirmation that your account is activated, you can " +"notify" +msgstr "" + +msgid "that you wish to join her/his group(s) through" +msgstr "" + +msgid "this link" +msgstr "" + +#, python-format +msgid "" +"%(inviter_name)s is a member of the following group(s):" +msgstr "" + +msgid "We look forward to seeing you on the platform," +msgstr "" + +msgid "The GeoNode team." +msgstr "" + +msgid "is inviting you to join" +msgstr "" + +msgid "To do so, please register at" +msgstr "" + +msgid "that you wish to join her/his group(s) through this link" +msgstr "" + +msgid "is a member of the following group(s)" +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Updating Metadata..." +msgstr "Edit Map Metadata" + +msgid "Topic Category is mandatory and cannot be empty!" +msgstr "" + +msgid "Group is mandatory and cannot be empty!" +msgstr "" + +msgid "Identification" +msgstr "" + +msgid "Yes" +msgstr "" + +msgid "Published" +msgstr "" + +msgid "email" +msgstr "" + +msgid "Information" +msgstr "" + +msgid "Identification Image" +msgstr "" + +msgid "Spatial Extent" +msgstr "" + +msgid "Projection System" +msgstr "" + +msgid "Extension x0" +msgstr "" + +msgid "Extension x1" +msgstr "" + +msgid "Extension y0" +msgstr "" + +msgid "Extension y1" +msgstr "" + +msgid "Features" +msgstr "" + +msgid "Contact Points" +msgstr "" + +msgid "References" +msgstr "" + +msgid "Link Online" +msgstr "" + +msgid "Metadata Page" +msgstr "" + +msgid "Online Link" +msgstr "" + +msgid "* Field declared Mandatory by the Metadata Schema" +msgstr "" + +msgid "Please, fill the following items before clicking on preview:" +msgstr "" + +msgid "Request perission" +msgstr "" + +msgid "Request editing for resource: " +msgstr "" + +msgid "Back to resource" +msgstr "" + +msgid "Send request" +msgstr "" + +msgid "Your account has been approved and is now active." +msgstr "" + +msgid "You can use the login form at" +msgstr "" + +msgid "Welcome at" +msgstr "" + +msgid "Your account is now active" +msgstr "" + +msgid "has requested access to the site." +msgstr "" + +msgid "" +"You can enable access by setting the user as active on the admin section" +msgstr "" + +msgid "A user has requested access to the site" +msgstr "" + +msgid "The following document was approved" +msgstr "" + +msgid "owned by" +msgstr "" + +msgid "You can visit the document's detail page here" +msgstr "" + +msgid "A document has been approved" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "A comment has been posted for the document" +msgstr "You do not have permission to view this document" + +msgid "by the user" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this document" +msgid "A comment has been posted for a document" +msgstr "You do not have permission to view this document" + +msgid "The user" +msgstr "" + +#, fuzzy +#| msgid "Add a Comment" +msgid "uploaded the following document" +msgstr "Add a Comment" + +msgid "A document has been uploaded" +msgstr "" + +msgid "The following document was deleted" +msgstr "" + +msgid "A document has been deleted" +msgstr "" + +msgid "The following document was published" +msgstr "" + +msgid "A document has been published" +msgstr "" + +msgid "rated" +msgstr "" + +msgid "the following document" +msgstr "" + +msgid "You can visit the layer's detail page here" +msgstr "" + +msgid "A document has been rated" +msgstr "" + +msgid "The following document was updated" +msgstr "" + +msgid "A document has been updated" +msgstr "" + +msgid "You have received the following notice from" +msgstr "" + +msgid "To change how you receive notifications, please go to " +msgstr "" + +#, python-format +msgid "[%(current_site)s] %(message)s" +msgstr "" + +#, fuzzy, python-format +#| msgid "Notices" +msgid "%(notice)s" +msgstr "Notices" + +msgid "The following layer was approved" +msgstr "" + +msgid "A layer has been approved" +msgstr "" + +#, fuzzy +#| msgid "You do not have permission to view this layer" +msgid "A comment has been posted for the layer" +msgstr "You do not have permission to view this layer" + +msgid "A comment has been posted for a layer" +msgstr "" + +msgid "uploaded the following layer" +msgstr "" + +msgid "A new layer has been uploaded" +msgstr "" + +msgid "The following layer was deleted" +msgstr "" + +msgid "A layer has been deleted" +msgstr "" + +msgid "The following layer was published" +msgstr "" + +msgid "A layer has been published" +msgstr "" + +msgid "the following layer" +msgstr "" + +msgid "A layer has been rated" +msgstr "" + +msgid "The following layer was updated" +msgstr "" + +msgid "A layer has been updated" +msgstr "" + +msgid "The following map was approved" +msgstr "" + +msgid "You can visit the map's detail page here" +msgstr "" + +msgid "A map has been approved" +msgstr "" + +msgid "A comment has been posted for the map" +msgstr "" + +msgid "A comment has been posted for a map" +msgstr "" + +msgid "created the following map" +msgstr "" + +msgid "A map has been created" +msgstr "" + +msgid "The following map was deleted" +msgstr "" + +msgid "A map has been deleted" +msgstr "" + +msgid "The following map was published" +msgstr "" + +msgid "A map has been published" +msgstr "" + +msgid "the following map" +msgstr "" + +msgid "A map has been rated" +msgstr "" + +msgid "The following map was updated" +msgstr "" + +msgid "A map has been updated" +msgstr "" + +#, fuzzy +#| msgid "Notice settings" +msgid "Notification Settings" +msgstr "Notice settings" + +#, python-format +msgid "" +"\n" +"

          \n" +" Note:\n" +" You do not have a verified email address to which notices can be " +"sent. Add one now.\n" +"

          \n" +" " +msgstr "" + +msgid "Notification Type" +msgstr "" + +msgid "requested you to download this resource" +msgstr "" + +msgid "" +"Please go to resource page and assign the download permissions if you wish" +msgstr "" + +msgid "A resource's download has been requested" +msgstr "" + +msgid "Privacy & Cookies Policy" +msgstr "" + +msgid "Privacy & Cookies Policy." +msgstr "" + +msgid "Following" +msgstr "" + +msgid "This user is not following anyone" +msgstr "" + +msgid "Followers" +msgstr "" + +msgid "No one is following this user" +msgstr "" + +msgid "Manage Connections" +msgstr "" + +msgid "Blocking" +msgstr "" + +msgid "Unfollow" +msgstr "" + +msgid "Follow" +msgstr "" + +msgid "Unblock" +msgstr "" + +msgid "Block" +msgstr "" + +msgid "No longer" +msgstr "" + +#, python-format +msgid "You're not %(status_slug)s anyone!" +msgstr "" + +msgid "Success" +msgstr "" + +msgid "You are " +msgstr "" + +msgid "no longer" +msgstr "" + +#, python-format +msgid "%(to_user_username)s's profile" +msgstr "" + +msgid "Responsibles" +msgstr "" + +msgid "Date begins after:" +msgstr "" + +msgid "yyyy-mm-dd" +msgstr "" + +msgid "Date ends before:" +msgstr "" + +msgid "Search by region" +msgstr "" + +#, fuzzy +#| msgid "Your selections" +msgid "Selection" +msgstr "Your selections" + +msgid "No list items selected. Use the selection fields to add." +msgstr "" + +msgid "Filters" +msgstr "" + +msgid "Datasets found" +msgstr "" + +msgid "Maps found" +msgstr "" + +msgid "Documents found" +msgstr "" + +msgid "found" +msgstr "" + +msgid "search" +msgstr "" + +msgid "Search by name" +msgstr "" + +msgid "Total" +msgstr "" + +msgid "Most recent" +msgstr "" + +msgid "Less recent" +msgstr "" + +msgid "A - Z" +msgstr "" + +msgid "Z - A" +msgstr "" + +msgid "Most popular" +msgstr "" + +msgid "Text" +msgstr "" + +msgid "Share This Dataset" +msgstr "" + +msgid "Share This" +msgstr "" + +msgid "Social Network Login Failure" +msgstr "" + +msgid "" +"An error occurred while attempting to login via your social network account." +msgstr "" + +msgid "Code" +msgstr "" + +msgid "Exception" +msgstr "" + +msgid "Account Connections" +msgstr "" + +msgid "" +"You can sign in to your account using any of the following already connected " +"third party accounts:" +msgstr "" + +msgid "" +"You currently have no social network accounts connected to this account." +msgstr "" + +msgid "Add a 3rd Party Account" +msgstr "" + +msgid "Login Cancelled" +msgstr "" + +#, python-format +msgid "" +"You decided to cancel logging in to our site using one of your existing " +"accounts. If this was a mistake, please proceed to sign in." +msgstr "" + +msgid "Signup" +msgstr "" + +msgid "Sign Up" +msgstr "" + +#, python-format +msgid "" +"You are about to use your %(provider_name)s account to login to\n" +"%(site_name)s. As a final step, please complete the following form:" +msgstr "" + +msgid "Sign in with" +msgstr "" + +msgid "Connect with" +msgstr "" + +msgid "Sign up with" +msgstr "" + +msgid "Message Inbox" +msgstr "" + +msgid "Messages" +msgstr "" + +msgid "Mark selected as read" +msgstr "እንደተነበበ ምልክት ያድርጉበት" + +msgid "Mark selected as unread" +msgstr "እንዳልተነበበ ምልክት አድርግ" + +msgid "Create Message" +msgstr "" + +msgid "To users" +msgstr "" + +msgid "To groups" +msgstr "" + +msgid "Subject" +msgstr "" + +msgid "Content" +msgstr "" + +msgid "Send message" +msgstr "" + +msgid "Back to Inbox" +msgstr "" + +msgid "Sent to" +msgstr "ይላኩ ለ" + +msgid "by" +msgstr "" + +msgid "You have no messages" +msgstr "" + +msgid "Send Reply" +msgstr "" + +msgid "GeoNode Themes Library" +msgstr "" + +msgid "Fill in this section with markdown" +msgstr "" + +msgid "Check this if the jumbotron background image already contains text" +msgstr "" + +msgid "Disabling this slide will hide it from the slide show" +msgstr "" + +msgid "Choose between using jumbotron background and slide show" +msgstr "" + +msgid "Empty content" +msgstr "" + +msgid "Could not access to uploaded data." +msgstr "" + +msgid "" +"One or more XML files was provided, but no matching files were found for " +"them." +msgstr "" + +msgid "" +"One or more SLD files was provided, but no matching files were found for " +"them." +msgstr "" + +#, fuzzy +#| msgid "Edit Map Metadata" +msgid "Layer already exists" +msgstr "Edit Map Metadata" + +msgid "Name already in use and overwrite is False" +msgstr "" + +msgid "Error configuring time:" +msgstr "" + +msgid "The GeoServer Import Session is no more available" +msgstr "" + +#, python-brace-format +msgid "Expected to find layer named '{name}' in geoserver" +msgstr "" + +msgid "Exception occurred while parsing the provided Metadata file." +msgstr "" + +msgid "" +"The UUID identifier from the XML Metadata is already in use in this system." +msgstr "" + +msgid "Error configuring Layer" +msgstr "" + +msgid "Invalid zip file detected" +msgstr "" + +msgid "Could not find any valid spatial file inside the uploaded zip" +msgstr "" + +msgid "Invalid kmz file detected" +msgstr "" + +msgid "Could not find any kml files inside the uploaded kmz" +msgstr "" + +msgid "Only one shapefile per zip is allowed" +msgstr "" + +msgid "kml files with more than one GroundOverlay are not supported" +msgstr "" + +msgid "Ground overlay image declared in kml file cannot be found" +msgstr "" + +msgid "Only one kml file per ZIP is allowed" +msgstr "" + +msgid "Only one kml file per kmz is allowed" +msgstr "" + +msgid "" +"You are trying to upload multiple GeoTIFFs without a valid 'indexer." +"properties' file." +msgstr "" + +msgid "Only one raster file per ZIP is allowed" +msgstr "" + +msgid "No multiple rasters allowed" +msgstr "" + +msgid "" +"To support the time step, you must enable the OGC_SERVER DATASTORE option" +msgstr "" + +#, python-brace-format +msgid "Unsupported file type: {e.message}" +msgstr "" + +#, python-brace-format +msgid "File does not exist: {absolute_base_file}" +msgstr "" + +#, python-brace-format +msgid "unknown item state: {task.state}" +msgstr "" + +msgid "error during import: {progress.get('message')}" +msgstr "" + +msgid "Could not find any valid Time Regex for the Mosaic files." +msgstr "" + +msgid "Unsupported DataBase for Mosaics!" +msgstr "" + +msgid "Upload Session invalid or no more accessible!" +msgstr "" + +msgid "Invalid permission level." +msgstr "" + +msgid "Permission Denied" +msgstr "" + +#, fuzzy +#~| msgid "Create a map" +#~ msgid "Create Layer" +#~ msgstr "Create a map" + +#, fuzzy +#~| msgid "Group details" +#~ msgid "apps explore" +#~ msgstr "Group details" + +#, fuzzy +#~| msgid "Group details" +#~ msgid "groups explore" +#~ msgstr "Group details" + +#, fuzzy +#~| msgid "Edit Map Metadata" +#~ msgid "layers append" +#~ msgstr "Edit Map Metadata" + +#, fuzzy +#~| msgid "Create a map" +#~ msgid "Save Map" +#~ msgstr "Create a map" + +#, fuzzy +#~| msgid "Thumbnail" +#~ msgid "Thumbnail saved" +#~ msgstr "Thumbnail" diff --git a/geonode/locale/am/LC_MESSAGES/djangojs.mo b/geonode/locale/am/LC_MESSAGES/djangojs.mo new file mode 100644 index 0000000..c559d48 Binary files /dev/null and b/geonode/locale/am/LC_MESSAGES/djangojs.mo differ diff --git a/geonode/locale/am/LC_MESSAGES/djangojs.po b/geonode/locale/am/LC_MESSAGES/djangojs.po new file mode 100644 index 0000000..b4479f7 --- /dev/null +++ b/geonode/locale/am/LC_MESSAGES/djangojs.po @@ -0,0 +1,672 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +msgid "" +msgstr "" +"Project-Id-Version: GeoNode\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2021-07-08 09:03+0300\n" +"PO-Revision-Date: 2016-08-23 13:57+0000\n" +"Last-Translator: Julien Collaer \n" +"Language-Team: Amharic (http://www.transifex.com/geonode/geonode/language/" +"am/)\n" +"Language: am\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" + +msgid "STORE_UPDATED" +msgstr "" + +#, javascript-format +msgid "Missing a %s file, which is required" +msgstr "" + +msgid "ESRI Shapefile" +msgstr "" + +msgid "ASCII Text File" +msgstr "" + +msgid "GeoTIFF" +msgstr "" + +msgid "Comma Separated Value" +msgstr "" + +msgid "Google Earth KML" +msgstr "" + +msgid "Google Earth KML with a GroundOverlay" +msgstr "" + +msgid "Google Earth KMZ" +msgstr "" + +msgid "GeoJSON" +msgstr "" + +msgid "Zip Archive" +msgstr "" + +msgid "ERDAS Imagine" +msgstr "" + +msgid "National Imagery Transmission Format" +msgstr "" + +msgid "Controlled Image Base-1 Meter" +msgstr "" + +msgid "Controlled Image Base-5 Meter" +msgstr "" + +msgid "Controlled Image Base-10 Meter" +msgstr "" + +msgid "CADRG-Global Navigation Chart" +msgstr "" + +msgid "CADRG-Jet Navigation Chart" +msgstr "" + +msgid "CADRG-Operational Navigation Chart" +msgstr "" + +msgid "CADRG-Tactical Pilotage Chart" +msgstr "" + +msgid "CADRG-Joint Operations Graphic" +msgstr "" + +msgid "CADRG-Topographic Line Map 100K" +msgstr "" + +msgid "CADRG-Topographic Line Map 50K" +msgstr "" + +msgid "JPEG2000" +msgstr "" + +msgid "MrSID-Multi-resolution Seamless Image Database" +msgstr "" + +msgid "XML Metadata File" +msgstr "" + +msgid "Style Layer Descriptor" +msgstr "" + +msgid "JPEG" +msgstr "" + +msgid "PNG" +msgstr "" + +msgid "GIF" +msgstr "" + +msgid "Unexpected error!" +msgstr "" + +msgid "" +" - 400 Bad Request. Server cannot or will not process the request due to " +"something that is perceived to be a client error (e.g., malformed request " +"syntax, invalid request message framing, or deceptive request routing)." +msgstr "" + +msgid "" +" - 401 Unauthorized. Request was not sent with the proper authentication " +"credentials." +msgstr "" + +msgid "" +" - 403 Forbidden. This is generally related to permission rules on your " +"server. Contact the system administrator for more information regarding this " +"error message." +msgstr "" + +msgid "" +" - 404 Not Found. Origin server was unable or unwilling to find the resource " +"requested." +msgstr "" + +msgid "" +" - 405 Method Not Allowed. Origin server is aware of the requested resource, " +"but the request method used is not supported." +msgstr "" + +msgid "" +" - 406 Not Acceptable. Resource is not available at the origin that adheres " +"to negotiation headers that were set prior (e.g. via 'Accept-Charset' and " +"'Accept-Language' headers)." +msgstr "" + +msgid "" +" - 407 Authentication Required. The client did not send the required " +"authentication with the request." +msgstr "" + +msgid "" +" - 408 Request Timeout. The origin server did not receive the complete " +"request in what it considers a reasonable time." +msgstr "" + +msgid "" +" - 409 Conflict. The request did not complete because of a conflict with the " +"current state of the resource. Typically happens on a PUT request where " +"multiple clients are attempting to edit the same resource." +msgstr "" + +msgid "" +" - 410 Gone. The resource requested is permanently missing at the origin." +msgstr "" + +msgid "" +" - 411 Length Required. Client did not define the 'Content-Length' of the " +"request body in the headers and this is required to obtain the resource." +msgstr "" + +msgid "" +" - 412 Precondition Failed. Server denies the request because the resource " +"failed to meet the conditions specified by the client." +msgstr "" + +msgid "" +" - 413 Payload Too Large. Refusal from the server to process the request " +"because the payload sent from the client is larger than the server wished to " +"accept. Server has the optional to close the connection." +msgstr "" + +msgid "" +" - 414 URI Too Long. Refusal from the server that the URI was too long to be " +"processed. For example, if a client is attempting a GET request with an " +"unusually long URI after a POST, this could be seen as a security risk and a " +"414 gets generated." +msgstr "" + +msgid "" +" - 415 Unsupported Media Type. Refusal from the server to process the format " +"of the current payload. One way to identify and fix this issue would be to " +"look at the 'Content-Type' or 'Content-Encoding' headers sent in the " +"client’s request." +msgstr "" + +msgid "" +" - 417 Expectation Failed. Failure of server to meet the requirements " +"specified in the 'Expect' header of the client’s request." +msgstr "" + +msgid "" +" - 429 Too Many Requests. Client has sent too many requests in the specified " +"amount of time according to the server." +msgstr "" + +msgid "" +" - 499 Client Close Request. Nginx specific response code to indicate when " +"the connection has been closed by the client while the server is still " +"processing its request, making server unable to send a status code back." +msgstr "" + +msgid "" +" - 500 Internal Server Error. This error indicates that the server has " +"encountered an unexpected condition. This often occurs when an application " +"request cannot be fulfilled due to the application being configured " +"incorrectly on the server." +msgstr "" + +msgid "" +" - 501 Not Implemented. This error indicates that the HTTP method sent by " +"the client is not supported by the server. This is most often caused by the " +"server being out of date. It is a very rare error and generally requires " +"that the web server be updated." +msgstr "" + +msgid "" +" - 502 Bad Gateway. This error is usually due to improperly configured proxy " +"servers. The first step in resolving the issue is to clear the client's " +"cache." +msgstr "" + +msgid "" +" - 503 Service Unavailable. This error occurs when the server is unable to " +"handle requests due to a temporary overload or due to the server being " +"temporarily closed for maintenance. The error indicates that the server will " +"only temporarily be down." +msgstr "" + +msgid "" +" - 504 Gateway Timeout. GeoNode lost the connection with GeoServer or DB due " +"to a connection timeout. Consider using the management commands to import " +"data!" +msgstr "" + +msgid "" +" - 505 HTTP Version Not Supported. This error occurs when the server refuses " +"to support the HTTP protocol that has been specified by the client computer. " +"This can be caused by the protocol not being specified properly by the " +"client computer; for example, if an invalid version number has been " +"specified." +msgstr "" + +msgid "" +" - 506 Variant Also Negotiates. This error indicates that the server is not " +"properly configured. Contact the system administrator to resolve this issue." +msgstr "" + +msgid "" +" - 507 Insufficient Storage. This error indicates that the server is out of " +"free memory. This is most likely to occur when an application that is being " +"requested cannot allocate the necessary system resources to run. To resolve " +"the issue, the server's hard disk may need to be cleaned of any unnecessary " +"documents to free up more hard disk space, its memory may need to be " +"expanded, or it may simply need to be restarted. Contact the system " +"administrator for more information regarding this error message." +msgstr "" + +msgid "" +" - 509 Bandwidth Limit Exceeded. This error occurs when the bandwidth limit " +"imposed by the system administrator has been reached. The only fix for this " +"issue is to wait until the limit is reset in the following cycle. Consult " +"the system administrator for information about acquiring more bandwidth." +msgstr "" + +msgid "" +" - 510 Not Extended. This error occurs when an extension attached to the " +"HTTP request is not supported by the web server. To resolve the issue, you " +"may need to update the server." +msgstr "" + +msgid "Unknwon" +msgstr "" + +msgid "" +" Error Code. Contact the system administrator for more information regarding " +"this error message." +msgstr "" + +msgid "Additional info: " +msgstr "" + +msgid "Finalizing" +msgstr "" + +msgid "Unexpected Error" +msgstr "" + +msgid "Your " +msgstr "" + +msgid " Info" +msgstr "" + +msgid "Edit Metadata" +msgstr "" + +msgid "Upload Metadata" +msgstr "" + +msgid "Upload SLD" +msgstr "" + +msgid "Manage Styles" +msgstr "" + +msgid "Please wait until GeoNode finished configuring it!" +msgstr "" + +msgid "The column %1 was renamed to %2
          " +msgstr "" + +msgid "Performing Final GeoServer Config Step" +msgstr "" + +msgid "You need to specify more information in order to complete your upload" +msgstr "" + +msgid "Files are ready to be ingested!" +msgstr "" + +msgid "A temporal dimension may be added to this Layer." +msgstr "" + +msgid "Performing GeoServer Config Step" +msgstr "" + +msgid "" +"
          Please check your network connection. In case of Layer Upload make sure " +"GeoServer is running and accepting connections." +msgstr "" + +msgid "
          " +msgstr "" + +msgid "
          Bad request or URL not found." +msgstr "" + +msgid "
          Unknown." +msgstr "" + +msgid "Layer files uploaded, configuring in GeoServer" +msgstr "" + +msgid "Remove" +msgstr "" + +#, javascript-format +msgid "The file %s is an unsupported file type, please select another file." +msgstr "" + +msgid "" +"You are trying to upload an incomplete set of files or not all mandatory " +"options have been validated.\n" +"\n" +"Please check for errors in the form!" +msgstr "" + +#, javascript-format +msgid "Available %s" +msgstr "Available %s" + +#, javascript-format +msgid "" +"This is the list of available %s. You may choose some by selecting them in " +"the box below and then clicking the \"Choose\" arrow between the two boxes." +msgstr "" +"This is the list of available %s. You may choose some by selecting them in " +"the box below and then clicking the \"Choose\" arrow between the two boxes." + +#, javascript-format +msgid "Type into this box to filter down the list of available %s." +msgstr "Type into this box to filter down the list of available %s." + +msgid "Filter" +msgstr "Filter" + +msgid "Choose all" +msgstr "Choose all" + +#, javascript-format +msgid "Click to choose all %s at once." +msgstr "Click to choose all %s at once." + +msgid "Choose" +msgstr "Choose" + +#, javascript-format +msgid "Chosen %s" +msgstr "Chosen %s" + +#, javascript-format +msgid "" +"This is the list of chosen %s. You may remove some by selecting them in the " +"box below and then clicking the \"Remove\" arrow between the two boxes." +msgstr "" +"This is the list of chosen %s. You may remove some by selecting them in the " +"box below and then clicking the \"Remove\" arrow between the two boxes." + +msgid "Remove all" +msgstr "Remove all" + +#, javascript-format +msgid "Click to remove all chosen %s at once." +msgstr "Click to remove all chosen %s at once." + +msgid "%(sel)s of %(cnt)s selected" +msgid_plural "%(sel)s of %(cnt)s selected" +msgstr[0] "94d7ef36d54e81afde607d8fd121772b_pl_0" +msgstr[1] "94d7ef36d54e81afde607d8fd121772b_pl_1" + +msgid "" +"You have unsaved changes on individual editable fields. If you run an " +"action, your unsaved changes will be lost." +msgstr "" +"You have unsaved changes on individual editable fields. If you run an " +"action, your unsaved changes will be lost." + +msgid "" +"You have selected an action, but you haven't saved your changes to " +"individual fields yet. Please click OK to save. You'll need to re-run the " +"action." +msgstr "" +"You have selected an action, but you haven't saved your changes to " +"individual fields yet. Please click OK to save. You'll need to re-run the " +"action." + +msgid "" +"You have selected an action, and you haven't made any changes on individual " +"fields. You're probably looking for the Go button rather than the Save " +"button." +msgstr "" +"You have selected an action, and you haven't made any changes on individual " +"fields. You're probably looking for the Go button rather than the Save " +"button." + +msgid "'" +msgstr "" + +msgid "-1" +msgstr "" + +msgid "trailingComments" +msgstr "" + +msgid "leadingComments" +msgstr "" + +msgid "innerComments" +msgstr "" + +msgid "Multiselect" +msgstr "" + +msgid "SelectList" +msgstr "" + +msgid "show:" +msgstr "" + +msgid "DraftEditor/root" +msgstr "" + +msgid "DraftEditor/editorContainer" +msgstr "" + +msgid "-" +msgstr "" + +msgid "uirevision" +msgstr "" + +msgid "bgcolor" +msgstr "" + +msgid "bordercolor" +msgstr "" + +msgid "borderwidth" +msgstr "" + +msgid "orientation" +msgstr "" + +msgid "traceorder" +msgstr "" + +msgid "tracegroupgap" +msgstr "" + +msgid "itemsizing" +msgstr "" + +msgid "itemclick" +msgstr "" + +msgid "itemdoubleclick" +msgstr "" + +msgid "x" +msgstr "" + +msgid "xanchor" +msgstr "" + +msgid "y" +msgstr "" + +msgid "yanchor" +msgstr "" + +msgid "valign" +msgstr "" + +msgid "title.text" +msgstr "" + +msgid "title.side" +msgstr "" + +msgid "visible" +msgstr "" + +msgid "thickness" +msgstr "" + +msgid "autorange" +msgstr "" + +msgid "range" +msgstr "" + +msgid "bingroup" +msgstr "" + +msgid "Print configuration broken (" +msgstr "" + +msgid "Print configuration not available (" +msgstr "" + +msgid "go" +msgstr "" + +msgid "goBack" +msgstr "" + +msgid "goForward" +msgstr "" + +msgid "Abort" +msgstr "" + +msgid "As Sibling" +msgstr "" + +msgid "As child" +msgstr "" + +#, fuzzy +#~| msgid "Now" +#~ msgid "No" +#~ msgstr "Now" + +#~ msgid "" +#~ "January February March April May June July August September October " +#~ "November December" +#~ msgstr "" +#~ "January February March April May June July August September October " +#~ "November December" + +#~ msgid "S M T W T F S" +#~ msgstr "S M T W T F S" + +#~ msgid "Show" +#~ msgstr "Show" + +#~ msgid "Hide" +#~ msgstr "Hide" + +#~ msgid "Clock" +#~ msgstr "Clock" + +#~ msgid "Choose a time" +#~ msgstr "Choose a time" + +#~ msgid "Midnight" +#~ msgstr "Midnight" + +#~ msgid "6 a.m." +#~ msgstr "6 a.m." + +#~ msgid "Noon" +#~ msgstr "Noon" + +#~ msgid "Today" +#~ msgstr "Today" + +#~ msgid "Calendar" +#~ msgstr "Calendar" + +#~ msgid "Yesterday" +#~ msgstr "Yesterday" + +#~ msgid "Tomorrow" +#~ msgstr "Tomorrow" + +#~ msgid "Add user..." +#~ msgstr "Add user..." + +#~ msgid "Who can view and download this data?" +#~ msgstr "Who can view and download this data?" + +#~ msgid "Anyone" +#~ msgstr "Anyone" + +#~ msgid "Any registered user" +#~ msgstr "Any registered user" + +#~ msgid "Only users who can edit" +#~ msgstr "Only users who can edit" + +#~ msgid "Who can edit this data?" +#~ msgstr "Who can edit this data?" + +#~ msgid "Only the following users or groups:" +#~ msgstr "Only the following users or groups:" + +#~ msgid "Who can manage and edit this data?" +#~ msgstr "Who can manage and edit this data?" + +#~ msgid "GeoTiff File" +#~ msgstr "GeoTiff File" + +#~ msgid "Comma Separated File" +#~ msgstr "Comma Separated File" + +#~ msgid "Zip Archives" +#~ msgstr "Zip Archives" + +#~ msgid "You are uploading an incomplete set of files." +#~ msgstr "You are uploading an incomplete set of files." + +#~ msgid "No error message supplied" +#~ msgstr "No error message supplied" diff --git a/geonode/locale/ar/LC_MESSAGES/django.mo b/geonode/locale/ar/LC_MESSAGES/django.mo new file mode 100644 index 0000000..d4e6db1 Binary files /dev/null and b/geonode/locale/ar/LC_MESSAGES/django.mo differ diff --git a/geonode/locale/ar/LC_MESSAGES/django.po b/geonode/locale/ar/LC_MESSAGES/django.po new file mode 100644 index 0000000..cb6f7e9 --- /dev/null +++ b/geonode/locale/ar/LC_MESSAGES/django.po @@ -0,0 +1,7979 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Translators: +# Andres Alfonso Lemon Gomez , 2014 +msgid "" +msgstr "" +"Project-Id-Version: GeoNode\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-02-22 10:29+0100\n" +"PO-Revision-Date: 2020-06-10 09:21+0200\n" +"Last-Translator: Julien Collaer \n" +"Language-Team: Arabic (http://www.transifex.com/geonode/geonode/language/" +"ar/)\n" +"Language: ar\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=6; plural=n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 " +"&& n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5;\n" +"X-Generator: Poedit 2.2.3\n" + +msgid "Request to download a resource" +msgstr "طلب تحميل مصدر" + +msgid "A request for downloading a resource was sent" +msgstr "تم ارسال طلب تنزيل المصدر" + +#, fuzzy +#| msgid "Resources" +msgid "Request resource change" +msgstr "موارد" + +#, fuzzy +#| msgid "A user has requested access to the site" +msgid "Owner has requested permissions to modify a resource" +msgstr "لقد طلب أحد المستخدمين الدخول إلى الموقع" + +msgid "series" +msgstr "سلسلة" + +msgid "computer program or routine" +msgstr "برنامج كمبيوتر أو روتين" + +msgid "feature type" +msgstr "نوع الميزة" + +msgid "copy or imitation of an existing or hypothetical object" +msgstr "نسخ أو تقليد كائن موجود أو افتراضى" + +msgid "collection hardware" +msgstr "أجهزة جمع" + +msgid "collection session" +msgstr "جلسة جمع" + +msgid "non-geographic data" +msgstr "البيانات غير الجغرافية" + +msgid "property type" +msgstr "نوع الملكية" + +msgid "field session" +msgstr "جلسة الحقل" + +msgid "dataset" +msgstr "مجموعة البيانات" + +msgid "service interfaces" +msgstr "واجهات خدمة" + +msgid "attribute class" +msgstr "فئة السمة" + +msgid "characteristic of a feature" +msgstr "سمة ميزة" + +msgid "tile or spatial subset of geographic data" +msgstr "البلاط أو مجموعة فرعية من البيانات المكانية الجغرافية" + +msgid "feature" +msgstr "ميزة" + +msgid "dimension group" +msgstr "مجموعة الأبعاد" + +msgid "frequency of maintenance for the data is not known" +msgstr "تردد الصيانة للبيانات غير معروف" + +msgid "data is repeatedly and frequently updated" +msgstr "يتم تحديث البيانات بشكل متكرر وكثيرا" + +msgid "there are no plans to update the data" +msgstr "لا توجد خطط لتحديث البيانات" + +msgid "data is updated each day" +msgstr "يتم تحديث البيانات كل يوم" + +msgid "data is updated every year" +msgstr "يتم تحديث البيانات كل عام" + +msgid "data is updated as deemed necessary" +msgstr "يتم تحديث البيانات بحسب ما تقتضيه الضرورة" + +msgid "data is updated each month" +msgstr "يتم تحديث البيانات كل شهر" + +msgid "data is updated every two weeks" +msgstr "يتم تحديث البيانات كل أسبوعين" + +msgid "data is updated in intervals that are uneven in duration" +msgstr "يتم تحديث البيانات في فترات متفاوتة في المدة" + +msgid "data is updated on a weekly basis" +msgstr "يتم تحديث البيانات بشكل أسبوعي" + +msgid "data is updated twice each year" +msgstr "يتم تحديث البيانات مرتين كل عام" + +msgid "data is updated every three months" +msgstr "يتم تحديث البيانات كل ثلاثة أشهر" + +msgid "No information provided" +msgstr "لم يتم تقديم أية معلومات" + +msgid "Category is required." +msgstr "الفئة مطلوبة." + +msgid "Keywords from Thesaurus" +msgstr "كلمات رئيسية من قاموس المرادفات" + +msgid "List of keywords from Thesaurus" +msgstr "قائمة الكلمات الرئيسية من قاموس المرادفات" + +msgid "Abstract" +msgstr "ملخص" + +msgid "Purpose" +msgstr "هدف" + +#, fuzzy +#| msgid "Other Settings" +msgid "Other constraints" +msgstr "اعدادات اخرى" + +#, fuzzy +#| msgid "supplemental information" +msgid "Supplemental information" +msgstr "معلومات إضافية" + +#, fuzzy +#| msgid "data quality statement" +msgid "Data quality statement" +msgstr "بيان جودة البيانات" + +msgid "Owner" +msgstr "مالك" + +msgid "Date" +msgstr "تاريخ" + +msgid "temporal extent start" +msgstr "بدء المدى الزمني" + +msgid "temporal extent end" +msgstr "نهاية المدى الزمني" + +msgid "Person outside GeoNode (fill form)" +msgstr "شخص خارج GeoNode (تعبئة نموذج)" + +msgid "Point of Contact" +msgstr "جهة الاتصال" + +msgid "Metadata Author" +msgstr "مؤلف البيانات الوصفية" + +msgid "Free-text Keywords" +msgstr "كلمات النص الحر" + +msgid "" +"A space or comma-separated list of keywords. Use the widget to select from " +"Hierarchical tree." +msgstr "" +"مساحة أو قائمة مفصولة بفواصل بكلمات مفتاحية. استخدم عنصر واجهة المستخدم " +"للتحديد من شجرة التسلسل الهرمي." + +msgid "Regions" +msgstr "المناطق" + +msgid "" +"Additional metadata, must be in format [ {\"metadata_key\": " +"\"metadata_value\"}, {\"metadata_key\": \"metadata_value" +"\"} ]" +msgstr "" + +msgid "Group" +msgstr "مجموعة" + +msgid "Category" +msgstr "فئة" + +msgid "License" +msgstr "رخصة" + +msgid "Language" +msgstr "لغة" + +#, fuzzy +#| msgid "Resources" +msgid "Resource" +msgstr "موارد" + +msgid "Reason" +msgstr "" + +msgid "Short reasoning behind the request" +msgstr "منطق مختصر وراء الطلب" + +msgid "function performed by the responsible party" +msgstr "الوظيفة التي تؤديها الجهة المسؤولة" + +msgid "brief narrative summary of the content of the resource(s)" +msgstr "ملخص سردي موجز لمحتوى المورد (الموارد)" + +msgid "reference date for the cited resource" +msgstr "تاريخ مرجعي للمورد المذكور" + +msgid "identification of when a given event occurred" +msgstr "تحديد عند وقوع حدث معين" + +msgid "version of the cited resource" +msgstr "نسخة من الموارد المذكورة" + +msgid "" +"authority or function assigned, as to a ruler, legislative assembly, " +"delegate, or the like." +msgstr "" + +msgid "a DOI will be added by Admin before publication." +msgstr "سيتم إضافة DOI من قبل المسؤول قبل النشر." + +msgid "summary of the intentions with which the resource(s) was developed" +msgstr "ملخص النوايا التي تم بها تطوير المصدر(المصادر)" + +msgid "" +"frequency with which modifications and deletions are made to the data after " +"it is first produced" +msgstr "" +"التردد الذي يتم به إجراء التعديلات والحذف على البيانات بعد إنتاجها لأول مرة" + +#, fuzzy +#| msgid "" +#| "commonly used word(s) or formalised word(s) or phrase(s) used to describe " +#| "the subject (space or comma-separated" +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated)" +msgstr "" +"كلمة (كلمات) شائعة الاستخدام أو كلمة (كلمات) رسمية أو عبارة (كلمات) مستخدمة " +"لوصف الموضوع (مسافة أو مفصولة بفواصل)" + +#, fuzzy +#| msgid "" +#| "formalised word(s) or phrase(s) from a fixed thesaurus used to describe " +#| "the subject (space or comma-separated" +msgid "" +"formalised word(s) or phrase(s) from a fixed thesaurus used to describe the " +"subject (space or comma-separated)" +msgstr "" +"كلمة (كلمات) رسمية أو عبارة (كلمات) من قاموس مرادفات ثابت يستخدم لوصف " +"الموضوع (مسافة أو مفصولة بفواصل)" + +msgid "keyword identifies a location" +msgstr "الكلمة تحدد مكان" + +msgid "limitation(s) placed upon the access or use of the data." +msgstr "القيد او القيود المفروضة على الوصول أو استخدام البيانات." + +msgid "" +"other restrictions and legal prerequisites for accessing and using the " +"resource or metadata" +msgstr "" +"قيود وشروط قانونية أخرى للوصول إلى واستخدام المورد أو البيانات التعريفية" + +msgid "license of the dataset" +msgstr "ترخيص مجموعة البيانات" + +msgid "language used within the dataset" +msgstr "اللغة المستخدمة في مجموعة البيانات" + +msgid "" +"high-level geographic data thematic classification to assist in the grouping " +"and search of available geographic data sets." +msgstr "" +"التصنيف المواضيعي للبيانات الجغرافية الرفيع المستوى للمساعدة في تجميع " +"مجموعات البيانات الجغرافية المتاحة والبحث عنها." + +msgid "method used to represent geographic information in the dataset." +msgstr "الطريقة المستخدمة لتمثيل المعلومات الجغرافية في مجموعة البيانات." + +msgid "time period covered by the content of the dataset (start)" +msgstr "الفترة الزمنية التي يغطيها محتوى مجموعة البيانات (البداية)" + +msgid "time period covered by the content of the dataset (end)" +msgstr "الفترة الزمنية التي يغطيها محتوى مجموعة البيانات (النهاية)" + +msgid "" +"general explanation of the data producer's knowledge about the lineage of a " +"dataset" +msgstr "شرح عام لمعرف منتجي البيانات حول نسب مجموعة البيانات" + +msgid "" +"Additional metadata, must be in format [ {\"metadata_key\": \"metadata_value" +"\"}, {\"metadata_key\": \"metadata_value\"} ]" +msgstr "" + +msgid "title" +msgstr "عنوان" + +msgid "name by which the cited resource is known" +msgstr "الاسم الذي يعرف به المورد المذكور" + +msgid "abstract" +msgstr "ملخص" + +msgid "purpose" +msgstr "هدف" + +msgid "date" +msgstr "تاريخ" + +msgid "date type" +msgstr "نوع التاريخ" + +msgid "edition" +msgstr "الإصدار" + +#, fuzzy +#| msgid "Attribute" +msgid "Attribution" +msgstr "سمة" + +msgid "DOI" +msgstr "‫DOI" + +msgid "maintenance frequency" +msgstr "تردد الصيانة" + +msgid "keywords" +msgstr "الكلمات الدالة" + +msgid "keywords region" +msgstr "منطقة الكلمات الرئيسية" + +msgid "restrictions" +msgstr "قيود" + +msgid "restrictions other" +msgstr "قيود أخرى" + +msgid "language" +msgstr "لغة" + +msgid "spatial representation type" +msgstr "نوع الحيز البياني" + +msgid "supplemental information" +msgstr "معلومات إضافية" + +msgid "any other descriptive information about the dataset" +msgstr "أي معلومات وصفية أخرى حول مجموعة البيانات" + +msgid "data quality statement" +msgstr "بيان جودة البيانات" + +msgid "CSW typename" +msgstr "نوع اسم CSW" + +msgid "CSW schema" +msgstr "مخطط CSW" + +msgid "CSW source" +msgstr "مصدر CSW" + +msgid "CSW insert date" +msgstr "تاريخ إدراج CSW" + +msgid "CSW type" +msgstr "نوع CSW" + +msgid "CSW anytext" +msgstr "CSW أي نص" + +msgid "CSW WKT geometry" +msgstr "هندسة CSW WKT" + +msgid "Metadata uploaded preserve" +msgstr "تم تحميل بيانات التعريف المحفوظة" + +msgid "Featured" +msgstr "متميز" + +msgid "Should this resource be advertised in home page?" +msgstr "هل يجب الإعلان عن هذا المورد في الصفحة الرئيسية؟" + +#, fuzzy +#| msgid "Is Published" +msgid "Was Published" +msgstr "تم نشره" + +msgid "Previous Published state." +msgstr "" + +msgid "Is Published" +msgstr "تم نشره" + +msgid "Should this resource be published and searchable?" +msgstr "هل ينبغي نشر هذه الموارد والبحث فيها؟" + +#, fuzzy +#| msgid "Map Approved" +msgid "Was Approved" +msgstr "خريطة موافق عليها" + +msgid "Previous Approved state." +msgstr "" + +msgid "Approved" +msgstr "موافق عليه" + +msgid "Is this resource validated from a publisher or editor?" +msgstr "هل تم التحقق من هذا المورد من ناشر أو محرر؟" + +msgid "Thumbnail url" +msgstr "رابط الصورة المصغرة" + +#, fuzzy +#| msgid "Thumbnail" +msgid "Thumbnail path" +msgstr "صورة مصغرة" + +#, fuzzy +#| msgid "Dirty State" +msgid "State" +msgstr "الدولة القذرة" + +msgid "Hold the resource processing state." +msgstr "" + +#, fuzzy +#| msgid "Resources" +msgid "Source Type" +msgstr "موارد" + +msgid "" +"The resource source type, which can be one of \"LOCAL\", \"REMOTE\" or " +"\"COPYREMOTE\"." +msgstr "" + +#, fuzzy +#| msgid "Remote Services" +msgid "Remote Service Typename" +msgstr "الخدمات عن بعد" + +#, fuzzy +#| msgid "Remove Remote Service" +msgid "Name of the Remote Service if any." +msgstr "إزالة الخدمة عن بعد" + +msgid "Dirty State" +msgstr "الدولة القذرة" + +msgid "Security Rules Are Not Synched with GeoServer!" +msgstr "لا تتم مزامنة قواعد الأمان مع GeoServer!" + +#, fuzzy +#| msgid "Resources" +msgid "Resource Type" +msgstr "موارد" + +msgid "Metadata" +msgstr "البيانات الوصفية" + +msgid "If true, will be excluded from search" +msgstr "" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "Extra Metadata" +msgstr "تعديل البيانات الوصفية" + +msgid "None" +msgstr "" + +msgid "View" +msgstr "View" + +msgid "Download" +msgstr "تحميل" + +msgid "Edit" +msgstr "تعديل" + +msgid "Manage" +msgstr "إدارة" + +msgid "For example \"kml\"" +msgstr "على سبيل المثال “kml”" + +msgid "For example \"View in Google Earth\"" +msgstr "على سبيل المثال \"عرض في Google Earth\"" + +msgid "For example \"text/xml\"" +msgstr "على سبيل المثال \"نص / XML\"" + +msgid "About" +msgstr "حول" + +msgid "Responsible, Point of Contact, Metadata Author" +msgstr "المالك، نقطة الاتصال، ‫مؤلف البيانات الوصفية" + +msgid "Responsible" +msgstr "الأطراف المسؤولة" + +msgid "Title" +msgstr "عنوان" + +msgid "SRID" +msgstr "رقم تعريف مرجعي لأنظمة الإحداثيات المكانية" + +msgid "For more info see" +msgstr "لمزيد من المعلومات راجع" + +#, fuzzy +#| msgid "Creating a Map" +msgid "Creation Date" +msgstr "إنشاء خريطة" + +#, fuzzy +#| msgid "publication" +msgid "Publication Date" +msgstr "نشرؤولض" + +#, fuzzy +#| msgid "Permission Denied" +msgid "Revision Date" +msgstr "طلب الاذن مرفوض" + +msgid "Type" +msgstr "نوع" + +msgid "Keywords" +msgstr "الكلمات الدالة" + +msgid "More info" +msgstr "المزيد من المعلومات" + +msgid "Maintenance Frequency" +msgstr "تردد الصيانة" + +msgid "Restrictions" +msgstr "قيود" + +msgid "Edition" +msgstr "الإصدار" + +msgid "Temporal Extent" +msgstr "المدى الزمني" + +msgid "Data Quality" +msgstr "جودة البيانات" + +msgid "Supplemental Information" +msgstr "معلومات إضافية" + +msgid "Spatial Representation Type" +msgstr "نوع التمثيل المكاني" + +msgid "No content created yet." +msgstr "لم يتم إنشاء محتوى بعد." + +msgid "Temporal Serie" +msgstr "سلسلة زمنية" + +msgid "Select" +msgstr "" + +msgid "Service is" +msgstr "الخدمة هي" + +msgid "online" +msgstr "متصل على الانترنت" + +msgid "offline" +msgstr "غير متصل على الانترنت" + +msgid "SECURITY NOT YET SYNCHRONIZED" +msgstr "الأمن لم يتزامن بعد" + +msgid "Sync permissions immediately" +msgstr "مزامنة الأذونات على الفور" + +msgid "PENDING APPROVAL" +msgstr "في انتظار الموافقة" + +msgid "UNPUBLISHED" +msgstr "غير منشورة" + +msgid "View Map" +msgstr "عرض الخريطة" + +msgid "Batch Edit" +msgstr "تعديل الدفعة" + +msgid "Cancel" +msgstr "إلغاء" + +msgid "Submit" +msgstr "قدم" + +msgid "Discover the available datasets." +msgstr "اكتشف مجموعات البيانات المتوفرة." + +#, fuzzy +#| msgid "Upload Document's Thumbnail" +msgid "Upload a Thumbnail" +msgstr "تحميل المستندات المصغرة" + +#, fuzzy +#| msgid "couldn't generate thumbnail" +msgid "Remove (and use auto generated thumbnail)" +msgstr "لا يمكن توليد الصورة المصغرة" + +msgid "Upload" +msgstr "رفع" + +#, fuzzy +#| msgid "couldn't generate thumbnail" +msgid "Auto generated thumbnail" +msgstr "لا يمكن توليد الصورة المصغرة" + +#, fuzzy +#| msgid "Featured Datasets" +msgid "Raster Dataset" +msgstr "مجموعات البيانات المميزة" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "Vector Dataset" +msgstr "تعديل البيانات الوصفية" + +#, fuzzy +#| msgid "Temporal Serie" +msgid "Vector Temporal Serie" +msgstr "سلسلة زمنية" + +#, fuzzy +#| msgid "Remove Layers" +msgid "Remote Dataset" +msgstr "إزالة الطبقات" + +#, fuzzy +#| msgid "Create Layer" +msgid "WMS Cascade Dataset" +msgstr "إنشاء طبقة" + +msgid "me" +msgstr "أنا" + +msgid "System message: A request to modify resource" +msgstr "رسالة النظام: طلب لتعديل المورد" + +#, fuzzy +#| msgid "A user has requested access to the site" +msgid "The resource owner has requested to modify the resource" +msgstr "لقد طلب أحد المستخدمين الدخول إلى الموقع" + +#, fuzzy +#| msgid "Resources" +msgid "Resource title" +msgstr "موارد" + +msgid "Reason for the request" +msgstr "سبب الطلب" + +msgid "" +"To allow the change, set the resource to not \"Approved\" under the metadata " +"settingsand write message to the owner to notify him" +msgstr "" +"للسماح بالتغيير، قم بتعيين المورد إلى \"تمت الموافقة عليه\" ضمن إعدادات " +"بيانات التعريف وكتابة رسالة إلى المالك لإخطاره" + +#, fuzzy +#| msgid "Service rescanned successfully" +msgid "Resource Cloned Successfully!" +msgstr "تم إعادة فحص الخدمة بنجاح" + +#, fuzzy, python-brace-format +#| msgid "Some error occurred while trying to access the uploaded schema: %s" +msgid "Error Occurred while Cloning the Resource: {e}" +msgstr "حدث بعض الأخطاء أثناء محاولة الوصول إلى المخطط الذي تم تحميله: %s" + +#, fuzzy +#| msgid "You are not permitted to view this document" +msgid "You are not permitted to view this resource" +msgstr "لا يسمح لك بعرض هذه الوثيقة" + +#, fuzzy +#| msgid "Not provided" +msgid "Not allowed" +msgstr "غير مزود" + +msgid "Resource Metadata" +msgstr "بيانات تعريف المورد" + +msgid "Thumbnail" +msgstr "صورة مصغرة" + +msgid "Resource ID" +msgstr "معرف المورد" + +msgid "Start" +msgstr "بداية" + +msgid "End" +msgstr "نهاية" + +msgid "Extent" +msgstr "أبعاد" + +msgid "Spatial Reference System Identifier" +msgstr "معرف النظام المرجعي المكاني" + +msgid "Attributes" +msgstr "سمات" + +msgid "Attribute name" +msgstr "اسم السمة" + +msgid "Description" +msgstr "وصف" + +msgid "GeoNode Client Library" +msgstr "مكتبة عميل GeoNode" + +msgid "" +"Flora and/or fauna in natural environment. Examples: wildlife, vegetation, " +"biological sciences, ecology, wilderness, sealife, wetlands, habitat." +msgstr "" +"النباتات و / أو الحيوانات في البيئة الطبيعية. أمثلة: الحياة البرية ، الغطاء " +"النباتي ، العلوم البيولوجية ، البيئة ، الحياة البرية ، الحياة البحرية، " +"الأراضي الرطبة ، مسكن طبيعي." + +msgid "Biota" +msgstr "الكائنات الحية" + +msgid "" +"Legal land descriptions. Examples: political and administrative boundaries." +msgstr "أوصاف الأراضي القانونية. أمثلة: الحدود السياسية والادارية." + +msgid "Boundaries" +msgstr "الحدود" + +msgid "" +"Processes and phenomena of the atmosphere. Examples: cloud cover, weather, " +"climate, atmospheric conditions, climate change, precipitation." +msgstr "" +"عمليات وظواهر الغلاف الجوي. أمثله: الغلاف السحابي ، الطقس ، المناخ ، الظروف " +"الجوية ، تغير المناخ ، هطول الامطار." + +msgid "Climatology Meteorology Atmosphere" +msgstr "" + +msgid "" +"Economic activities, conditions and employment. Examples: production, " +"labour, revenue, commerce, industry, tourism and ecotourism, forestry, " +"fisheries, commercial or subsistence hunting, exploration and exploitation " +"of resources such as minerals, oil and gas." +msgstr "" +"الأنشطة الاقتصادية والظروف والعمالة. أمثلة: الإنتاج ، العمل ، الإيرادات ، " +"التجارة ، الصناعة ، السياحة والسياحة البيئية ، الغابات، مصايد الأسماك ، " +"الصيد التجاري أو صيد الكفاف ، استكشاف واستغلال موارد مثل المعادن والنفط " +"والغاز." + +msgid "Economy" +msgstr "اقتصاد" + +msgid "" +"Height above or below sea level. Examples: altitude, bathymetry, digital " +"elevation models, slope, derived products." +msgstr "" +"ارتفاع فوق أو تحت مستوي سطح البحر. أمثله: الارتفاع ، قياس الأعماق ، نماذج " +"الارتفاع الرقمي ، الانحدار ، المنتجات المشتقة." + +msgid "Elevation" +msgstr "ارتفاع" + +msgid "" +"Environmental resources, protection and conservation. Examples: " +"environmental pollution, waste storage and treatment, environmental impact " +"assessment, monitoring environmental risk, nature reserves, landscape." +msgstr "" +"الموارد البيئية، الحماية والحفظ. أمثلة: التلوث البيئي، تخزين النفايات " +"ومعالجتها، تقييم الأثر البيئي، مراقبة المخاطر البيئية، المحميات الطبيعية، " +"المناظر الطبيعية." + +msgid "Environment" +msgstr "بيئة" + +msgid "" +"Rearing of animals and/or cultivation of plants. Examples: agriculture, " +"irrigation, aquaculture, plantations, herding, pests and diseases affecting " +"crops and livestock." +msgstr "" +"تربية الحيوانات و / أو زراعة النباتات. أمثلة: الزراعة, الري, تربية الأحياء " +"المائية, المزارع, الرعي, الآفات والأمراض التي تصيب المحاصيل والماشية." + +msgid "Farming" +msgstr "زراعة" + +msgid "" +"Information pertaining to earth sciences. Examples: geophysical features and " +"processes, geology, minerals, sciences dealing with the composition, " +"structure and origin of the earth s rocks, risks of earthquakes, volcanic " +"activity, landslides, gravity information, soils, permafrost, hydrogeology, " +"erosion." +msgstr "" +"المعلومات المتعلقة بعلوم الأرض. أمثله: الخصائص والعمليات الجيوفيزيائية ، " +"الجيولوجيا ، المعادن ، العلوم التي تتعامل مع التكوين, هيكل ومنشا صخور " +"الأرض ، مخاطر الزلازل ، النشاط البركاني ، الانهيالات الارضيه ، ومعلومات " +"الجاذبية ، التربة ، الأراضي المتجمدة ، الجيولوجيا المائية ، التعرية." + +#, fuzzy +#| msgid "Supplemental Information" +msgid "Geoscientific Information" +msgstr "معلومات إضافية" + +msgid "" +"Health, health services, human ecology, and safety. Examples: disease and " +"illness, factors affecting health, hygiene, substance abuse, mental and " +"physical health, health services." +msgstr "" +"الصحة ، الخدمات الصحية ، البيئة البشرية ، والسلامة. أمثلة: الأمراض والعلل، " +"العوامل التي تؤثر على الصحة ، النظافة ، تعاطي المخدرات ، الصحة العقلية " +"والبدنية ، الخدمات الصحية." + +msgid "Health" +msgstr "الصحة" + +msgid "" +"Base maps. Examples: land cover, topographic maps, imagery, unclassified " +"images, annotations." +msgstr "" +"الخرائط الأساسية. أمثلة: الغطاء الأرضي ، الخرائط الطبوغرافية ، الصور ، الصور " +"غير المصنفة ، التعليقات التوضيحية." + +msgid "Imagery Base Maps Earth Cover" +msgstr "" + +#, fuzzy +#| msgid "" +#| "Inland water features, drainage systems and their characteristics. " +#| "Examples: rivers and glaciers, salt lakes, water utilization plans, dams, " +#| "currents, floods, water quality, hydrographic charts." +msgid "" +"Inland water features, drainage systems and their characteristics. Examples: " +"rivers and glaciers, salt lakes, water utilization plans, dams, currents, " +"floods, water quality, hydrographic charts." +msgstr "" +"ميزات المياه الداخلية ، شبكات الصرف الصحي وخصائصها. أمثلة: الأنهار والأنهار " +"الجليدية ، البحيرات المالحة ، خطط استخدام المياه ، السدود ، التيارات ، " +"الفيضانات ، جودة المياه ، المخططات الهيدروغرافية." + +msgid "Inland Waters" +msgstr "المياه الداخلية" + +msgid "" +"Military bases, structures, activities. Examples: barracks, training " +"grounds, military transportation, information collection." +msgstr "" +"القواعد العسكرية، الهياكل، الأنشطة. أمثلة: الثكنات ، أماكن التدريب ، النقل " +"العسكري ، جمع المعلومات." + +#, fuzzy +#| msgid "Intelligence" +msgid "Intelligence Military" +msgstr "المخابرات" + +msgid "" +"Positional information and services. Examples: addresses, geodetic networks, " +"control points, postal zones and services, place names." +msgstr "" +"المعلومات والخدمات الموضعية. أمثلة: العناوين ، الشبكات الجيوديسية ، نقاط " +"التحكم ، المناطق والخدمات البريدية ، أسماء الأماكن." + +msgid "Location" +msgstr "موقع" + +msgid "" +"Features and characteristics of salt water bodies (excluding inland waters). " +"Examples: tides, tidal waves, coastal information, reefs." +msgstr "" +"ميزات وخصائص المسطحات المائية المالحة (باستثناء المياه الداخلية). أمثلة: " +"المد والجزر والأمواج المدية والمعلومات الساحلية والشعاب المرجانية." + +msgid "Oceans" +msgstr "المحيطات" + +msgid "" +"Information used for appropriate actions for future use of the land. " +"Examples: land use maps, zoning maps, cadastral surveys, land ownership." +msgstr "" +"المعلومات المستخدمة لاتخاذ الإجراءات المناسبة للاستخدام المستقبلي للأرض. " +"أمثلة: خرائط استخدام الأراضي ، خرائط تقسيم المناطق ، المسوحات المساحية ، " +"ملكية الأرض." + +#, fuzzy +#| msgid "Planning" +msgid "Planning Cadastre" +msgstr "مخطط" + +msgid "" +"Settlements, anthropology, archaeology, education, traditional beliefs, " +"manners and customs, demographic data, recreational areas and activities, " +"social impact assessments, crime and justice, census information. Economic " +"activities, conditions and employment." +msgstr "" +"المستوطنات ، الأنثروبولوجيا ، علم الآثار ، التعليم ، المعتقدات التقليدية ، " +"الآداب والعادات ، البيانات الديموغرافية ، المناطق والأنشطة الترفيهية ، " +"تقييمات الأثر الاجتماعي ، الجريمة والعدالة ، معلومات التعداد. الأنشطة " +"الاقتصادية والظروف والعمالة." + +msgid "Population" +msgstr "تعداد السكان" + +msgid "" +"Characteristics of society and cultures. Examples: settlements, " +"anthropology, archaeology, education, traditional beliefs, manners and " +"customs, demographic data, recreational areas and activities, social impact " +"assessments, crime and justice, census information." +msgstr "" +"خصائص المجتمع والثقافات. أمثلة: المستوطنات, الأنثروبولوجيا, علم الآثار, " +"التعليم, المعتقدات التقليدية, الأخلاق والعادات, البيانات الديموغرافية, " +"المناطق والأنشطة الترفيهية, تقييم التأثيرات الاجتماعية, الجريمة والعدالة, " +"معلومات التعداد." + +msgid "Society" +msgstr "المجتمع" + +msgid "" +"Man-made construction. Examples: buildings, museums, churches, factories, " +"housing, monuments, shops, towers." +msgstr "" +"بناء من صنع الإنسان. أمثلة: المباني ، المتاحف ، الكنائس ، المصانع ، " +"الإسكان ، الآثار ، المتاجر ، الأبراج." + +msgid "Structure" +msgstr "تركيب" + +msgid "" +"Means and aids for conveying persons and/or goods. Examples: roads, airports/" +"airstrips, shipping routes, tunnels, nautical charts, vehicle or vessel " +"location, aeronautical charts, railways." +msgstr "" +"الوسائل والمساعدات لنقل الأشخاص و/أو السلع. أمثله: الطرق, المطارات/مهابط " +"الرحلات, طرق الشحن, الانفاق, الخرائط الملاحية, موقع المركبات أو السفن, " +"الرسوم البيانية للطيران, السكك الحديدية." + +msgid "Transportation" +msgstr "وسائل النقل" + +msgid "" +"Energy, water and waste systems and communications infrastructure and " +"services. Examples: hydroelectricity, geothermal, solar and nuclear sources " +"of energy, water purification and distribution, sewage collection and " +"disposal, electricity and gas distribution, data communication, " +"telecommunication, radio, communication networks." +msgstr "" +"أنظمة الطاقة والمياه والنفايات والبنية التحتية والخدمات. أمثلة: الطاقة " +"الكهرمائية ، والطاقة الحرارية الأرضية ، ومصادر الطاقة الشمسية والنووية ، " +"وتنقية المياه وتوزيعها ، وجمع مياه الصرف الصحي والتخلص منها ، وتوزيع " +"الكهرباء والغاز ، واتصالات البيانات ، والاتصالات ، والراديو ، وشبكات " +"الاتصالات." + +msgid "Utilities Communication" +msgstr "" + +msgid "Document Created" +msgstr "تم إنشاء المستند" + +msgid "A Document was created" +msgstr "تم إنشاء مستند مسبقا" + +msgid "Document Updated" +msgstr "تم تحديث المستند‬" + +msgid "A Document was updated" +msgstr "تم تحديث المستند مسبقا" + +msgid "Document Approved" +msgstr "وثيقة معتمدة" + +msgid "A Document was approved by a Manager" +msgstr "تمت الموافقة على الوثيقة من قبل المدير" + +msgid "Document Published" +msgstr "وثيقة نشرت" + +msgid "A Document was published" +msgstr "تم نشر وثيقة" + +msgid "Document Deleted" +msgstr "تم حذف المستند" + +msgid "A Document was deleted" +msgstr "تم حذف المستند مسبقا" + +msgid "Comment on Document" +msgstr "التعليق على المستند" + +msgid "A Document was commented on" +msgstr "تم التعليق على الوثيقة" + +msgid "Rating for Document" +msgstr "تصنيف المستند" + +msgid "A rating was given to a document" +msgstr "تم إعطاء تصنيف للمستند" + +msgid "Width" +msgstr "عرض" + +msgid "Height" +msgstr "ارتفاع" + +msgid "Make" +msgstr "افعل" + +msgid "Model" +msgstr "نموذج" + +msgid "Latitude" +msgstr "خط العرض" + +msgid "Not specified" +msgstr "غير محدد" + +msgid "Longitude" +msgstr "خط الطول" + +msgid "Flash" +msgstr "وميض" + +msgid "True" +msgstr "صحيح" + +msgid "False" +msgstr "زائف" + +msgid "Speed Rating" +msgstr "تقييم سريع" + +msgid "" +"File size size exceeds {filesizeformat(max_size)}. Please try again with a " +"smaller file." +msgstr "" + +msgid "Link to" +msgstr "رابط ل" + +msgid "File" +msgstr "ملف" + +msgid "Permissions must be valid JSON." +msgstr "يجب أن تكون الأذونات JSON صالحة." + +msgid "Document must be a file or url." +msgstr "يجب أن يكون المستند ملفًا أو رابطًا." + +msgid "A document cannot have both a file and a url." +msgstr "لا يمكن أن يحتوي المستند على ملف ورابط." + +msgid "This file type is not allowed" +msgstr "لا يسمح هذا النوع من الملفات" + +#, fuzzy +#| msgid "Document must be a file or url." +msgid "Document must be a file." +msgstr "يجب أن يكون المستند ملفًا أو رابطًا." + +msgid "The URL of the document if it is external." +msgstr "عنوان المستند إذا كان خارجيًا." + +msgid "URL" +msgstr "رابط" + +#, fuzzy +#| msgid "Edit Metadata" +msgid "View Metadata" +msgstr "تعديل البيانات الوصفية" + +#, fuzzy +#| msgid "Download" +msgid "View and Download" +msgstr "تحميل" + +#, python-format +msgid "for %(document_title)s" +msgstr "إلى %(document_title)s" + +msgid "Completeness" +msgstr "كمال" + +msgid "Metadata Schema mandatory fields completed" +msgstr "اكتمال حقول بيانات التعريف الوصفية" + +msgid "Check Schema mandatory fields" +msgstr "تحقق مخطط الحقول الإلزامية" + +msgid "Error updating metadata. Please check the following fields: " +msgstr "خطأ تحديث البيانات الوصفية. يرجى التحقق من الحقول التالية " + +msgid "Done" +msgstr "منجز" + +msgid "Metadata Provider" +msgstr "مزود بيانات التعريف" + +msgid "<< Back" +msgstr "<< رجوع" + +msgid "Update" +msgstr "تحديث" + +msgid "Next >>" +msgstr "التالي >>" + +msgid "Select an option" +msgstr "حدد خيار" + +msgid "Explore Documents" +msgstr "تصفح المستندات" + +msgid "Edit Metadata" +msgstr "تعديل البيانات الوصفية" + +msgid "Editing details for" +msgstr "تعديل تفاصيل ل" + +msgid "Return to Document" +msgstr "العودة إلى المستند" + +msgid "Replace Document" +msgstr "استبدال الوثيقة" + +msgid "Replace " +msgstr "استبدل " + +msgid "Clear" +msgstr "مسح" + +msgid "Upload Document's Thumbnail" +msgstr "تحميل المستندات المصغرة" + +msgid "Drop files here" +msgstr "قم بوضع الملفات هنا" + +msgid " or select them one by one:" +msgstr " أو حددهم واحدا تلو الآخر:" + +msgid "Choose Files" +msgstr "اختيار المستخدمين" + +msgid "Files to be uploaded" +msgstr "الملفات المراد تحميلها" + +msgid "Is Upload Metadata XML Form" +msgstr "يتم تحميل بيانات التعريف لنموذج XML" + +msgid "Upload files" +msgstr "تحميل الملفات" + +msgid "Upload Document" +msgstr "تحميل الوثيقة" + +msgid "Permissions" +msgstr "أذونات" + +msgid "Upload Documents" +msgstr "تحميل المستندات" + +msgid "Allowed document types" +msgstr "أنواع المستندات المسموح بها" + +msgid "Done!" +msgstr "منجز!" + +msgid "Settings" +msgstr "الإعدادات" + +#, fuzzy +#| msgid "Metadata" +msgid "Advanced Metadata" +msgstr "البيانات الوصفية" + +msgid "Mandatory" +msgstr "إلزامي" + +msgid "Optional" +msgstr "اختياري" + +msgid "Basic Metadata" +msgstr "البيانات الوصفية الأساسية" + +msgid "Location and Licenses" +msgstr "الموقع والتراخيص" + +#, fuzzy +#| msgid "Resource Metadata" +msgid "Required Metadata" +msgstr "بيانات تعريف المورد" + +msgid "Optional Metadata" +msgstr "بيانات وصفية اختيارية" + +msgid "Choose one of the following..." +msgstr "اختر أحد الخيارات التالية..." + +msgid "Other, Optional, Metadata" +msgstr "أخرى ، اختياري ، بيانات التعريف" + +msgid "Responsible Parties" +msgstr "الأطراف المسؤولة" + +#, fuzzy +#| msgid "Owner and Permissions" +msgid "Responsible and Permissions" +msgstr "المالك الصلاحيات" + +msgid "Publishing" +msgstr "نشر" + +msgid "You are not allowed to view this document." +msgstr "لا يسمح لك بعرض هذه الوثيقة." + +msgid "You are not permitted to delete this document" +msgstr "لا يسمح لك بحذف هذه الوثيقة" + +msgid "You do not have permissions for this document." +msgstr "لم يكن لديك صلاحيات لهذه الوثيقة." + +msgid "You are not permitted to modify this document" +msgstr "لا يسمح لك بتعديل هذه الوثيقة" + +msgid "You are not permitted to modify this document's metadata" +msgstr "غير مسموح لك بتعديل البيانات الأولية لهذا المستند" + +msgid "You are not permitted to view this document" +msgstr "لا يسمح لك بعرض هذه الوثيقة" + +#, fuzzy +#| msgid "Maps found" +msgid "Not found" +msgstr "خرائط وجدت" + +msgid "You must set a point of contact for this resource" +msgstr "يجب تعيين نقطة اتصال لهذا المورد" + +msgid "You must set an author for this resource" +msgstr "يجب تعيين مؤلف لهذا المورد" + +msgid "Favorite" +msgstr "مفضلة" + +msgid "Add to Favorites" +msgstr "اضافة الى المفضلة" + +#, fuzzy +#| msgid "Add to Favorites" +msgid "Go to Favorites" +msgstr "اضافة الى المفضلة" + +msgid "Log in to add/delete Favorites." +msgstr "الدخول للاضافة / حذف المفضلات ." + +msgid "Delete from Favorites" +msgstr "حذف من المفضلات" + +msgid "Favorites" +msgstr "المفضلات‬" + +msgid "Favorites for" +msgstr "المفضلات ل" + +msgid "Item" +msgstr "" + +#, fuzzy +#| msgid "Map Created" +msgid "App Created" +msgstr "تم إنشاء الخريطة" + +#, fuzzy +#| msgid "A Map was created" +msgid "A App was created" +msgstr "تم إنشاء خريطة" + +#, fuzzy +#| msgid "Map Updated" +msgid "App Updated" +msgstr "تم تحديث الخريطة" + +#, fuzzy +#| msgid "A Map was updated" +msgid "A App was updated" +msgstr "تم تحديث الخريطة مسبقا" + +#, fuzzy +#| msgid "Map Approved" +msgid "App Approved" +msgstr "خريطة موافق عليها" + +#, fuzzy +#| msgid "A Map was approved by a Manager" +msgid "A App was approved by a Manager" +msgstr "تمت الموافقة على الخريطة من قبل المدير" + +#, fuzzy +#| msgid "Map Published" +msgid "App Published" +msgstr "الخريطة نشرت" + +#, fuzzy +#| msgid "A Map was published" +msgid "A App was published" +msgstr "تم نشر خريطة" + +#, fuzzy +#| msgid "Map Deleted" +msgid "App Deleted" +msgstr "تم حذف الخريطة" + +#, fuzzy +#| msgid "A Map was deleted" +msgid "A App was deleted" +msgstr "تم حذف الخريطة مسبقا" + +#, fuzzy +#| msgid "Comment on Map" +msgid "Comment on App" +msgstr "التعليق على خريطة" + +#, fuzzy +#| msgid "A map was commented on" +msgid "An App was commented on" +msgstr "وقد علق على خريطة" + +#, fuzzy +#| msgid "Rating for Map" +msgid "Rating for App" +msgstr "تصنيف للخريطة" + +#, fuzzy +#| msgid "A rating was given to a map" +msgid "A rating was given to an App" +msgstr "وقدم تصنيف للخريطة" + +msgid "Name" +msgstr "اسم" + +#, python-format +msgid "%(GEONODE_APPS_NAME)s" +msgstr "" + +#, python-format +msgid "for %(map_title)s" +msgstr "لـ%(map_title)s" + +#, fuzzy +#| msgid "" +#| "Note: this map's orginal metadata was populated by importing a metadata " +#| "XML file.\n" +#| " GeoNode's metadata import supports a subset of ISO, FGDC, and " +#| "Dublin Core metadata elements.\n" +#| " Some of your original metadata may have been lost." +msgid "" +"Note: this geoapp's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" +"ملاحظة: تم ملء البيانات الوصفية الأصلية لهذه الخريطة عن طريق استيراد ملف XML " +"شامل للبيانات الوصفية.\n" +"‬\n" +"‫         يدعم استيراد البيانات الوصفية ل GeoNode مجموعة فرعية من عناصر " +"البيانات الوصفية ISO, FGDC, and Dublin Core.\n" +"‬\n" +"‫         ربما فقدت بعض البيانات الوصفية الأصلية." + +#, fuzzy +#| msgid "Explore Maps" +msgid "Explore GeoApps" +msgstr "تصفح الخرائط" + +#, python-format +msgid "" +"\n" +" Editing details for %(map_title)s\n" +" " +msgstr "" +"\n" +"‫‫‫تعديل التفاصيل ل%(map_title)s" + +msgid "Return to" +msgstr "العودة إلى" + +msgid "New Map" +msgstr "خريطة جديدة" + +msgid "Other Settings" +msgstr "اعدادات اخرى" + +#, fuzzy +#| msgid "You are not permitted to delete this map." +msgid "You are not permitted to delete this app." +msgstr "لا يسمح لك بحذف هذه الخريطة." + +#, fuzzy +#| msgid "You do not have permissions for this map." +msgid "You do not have permissions for this app." +msgstr "ليس لديك صلاحيات لهذه الخريطة." + +#, fuzzy +#| msgid "You must be logged in to save this map" +msgid "You must be logged in to save this app" +msgstr "يجب أن تكون مسجلا لحفظ هذه الخريطة" + +#, fuzzy +#| msgid "You are not permitted to save or edit this map." +msgid "You are not permitted to save or edit this app." +msgstr "لا يسمح لك بحفظ أو تعديل هذه الخريطة." + +#, fuzzy +#| msgid "You are not allowed to modify this map's metadata." +msgid "You are not allowed to modify this app's metadata." +msgstr "لا يسمح لك بتعديل البيانات الوصفية هذه الخريطة ل." + +#, fuzzy +#| msgid "You are not allowed to view this map." +msgid "You are not allowed to view this app." +msgstr "لا يسمح لك بعرض هذه الخريطة." + +msgid "An unknown error has occured." +msgstr "حدث خطأ غير معروف." + +#, fuzzy +#| msgid "Layer Uploaded" +msgid "Dataset Uploaded" +msgstr "طبقة محملة" + +msgid "A layer was uploaded" +msgstr "تم رفع طبقة" + +#, fuzzy +#| msgid "Comment on Layer" +msgid "Comment on Dataset" +msgstr "التعليق على طبقة" + +msgid "A layer was commented on" +msgstr "تم التعليق على طبقة" + +#, fuzzy +#| msgid "Rating for Layer" +msgid "Rating for Dataset" +msgstr "تصنيف لطبقة" + +msgid "A rating was given to a layer" +msgstr "أعطي تصنيف لطبقة" + +#, fuzzy +#| msgid "Contact Points" +msgid "Points" +msgstr "نقاط الإتصال" + +msgid "Lines" +msgstr "" + +msgid "Polygons" +msgstr "" + +#, fuzzy +#| msgid "dataset" +msgid "Dataset name" +msgstr "مجموعة البيانات" + +#, fuzzy +#| msgid "Dataset Attributes" +msgid "Dataset title" +msgstr "سمات مجموعة البيانات" + +msgid "Geometry type" +msgstr "نوع الهندسة" + +#, fuzzy +#| msgid "Featured Datasets" +msgid "Create Dataset" +msgstr "مجموعات البيانات المميزة" + +#, fuzzy +#| msgid "Create an empty layer" +msgid "Create an empty dataset" +msgstr "إنشاء طبقة فارغة" + +msgid "Add Attribute" +msgstr "إضافة سمة" + +msgid "Create" +msgstr "إنشاء" + +msgid "String" +msgstr "" + +#, fuzzy +#| msgid "Intervals" +msgid "Integer" +msgstr "فترات" + +msgid "Float" +msgstr "" + +#, fuzzy +#| msgid "Not provided" +msgid "No title provided" +msgstr "غير مزود" + +msgid "No abstract provided" +msgstr "تجريد غير مقدم" + +msgid "An exception occurred loading data to PostGIS" +msgstr "حدث اعتراض أثناء تحميل البيانات إلى PostGIS" + +msgid " Additionally an error occured during database cleanup" +msgstr " بالإضافة إلى ذلك حدث خطأ أثناء تنظيف قاعدة البيانات" + +msgid "Invalid Projection. Dataset is missing CRS!" +msgstr "" + +msgid "GeoTIFF" +msgstr "GeoTIFF" + +msgid "Zipped Shapefile" +msgstr "Shapefile مضغوط" + +msgid "GML 2.0" +msgstr "GML 2.0" + +msgid "GML 3.1.1" +msgstr "GML 3.1.1" + +msgid "CSV" +msgstr "CSV" + +msgid "Excel" +msgstr "Excel" + +msgid "GeoJSON" +msgstr "GeoJSON أيضا" + +msgid "JPEG" +msgstr "JPEG" + +msgid "PDF" +msgstr "PDF" + +msgid "PNG" +msgstr "PNG" + +msgid "You don't have permissions to change style for this layer" +msgstr "لم يكن لديك صلاحيات لتغيير نمط هذه الطبقة" + +msgid "Bad HTTP Authorization Credentials." +msgstr "بيانات اعتماد تفويض HTTP غير صالحة." + +msgid "" +"a short version of the name consisting only of letters, numbers, underscores " +"and hyphens." +msgstr "" +"نسخة قصيرة من الاسم تتكون فقط من الحروف والأرقام والشرطات السفلية والواصلات." + +msgid "A group already exists with that slug." +msgstr "مجموعة موجودة بالفعل مع هذه السبيكة." + +msgid "A group already exists with that name." +msgstr "مجموعة موجودة بالفعل بهذا الاسم." + +msgid "User Identifiers" +msgstr "معرفات المستخدم" + +msgid "Assign manager role" +msgstr "إسناد دور للمدير" + +#, python-format +msgid "" +"The following are not valid usernames: %(errors)s; not added to the group" +msgstr "التالية ليست أسماء مستخدمين صالحة: %(errors)s; لا تضاف إلى المجموعة" + +msgid "Group Categories" +msgstr "فئات المجموعات" + +msgid "Public" +msgstr "عام" + +msgid "Public (invite-only)" +msgstr "عام (دعوة فقط)" + +msgid "Private" +msgstr "خاص" + +msgid "" +"Public: Any registered user can view and join a public group.
          Public " +"(invite-only):Any registered user can view the group. Only invited users " +"can join.
          Private: Registered users cannot see any details about the " +"group, including membership. Only invited users can join." +msgstr "" +"عام: يمكن لأي مستخدم مسجل عرض مجموعة عامة والانضمام إليها.
          عام " +"(للمدعوين فقط): يمكن لأي مستخدم مسجل عرض المجموعة. يمكن للمستخدمين المدعوين " +"فقط الانضمام.
          خاص: لا يمكن للمستخدمين المسجلين رؤية أي تفاصيل حول " +"المجموعة، بما في ذلك العضوية. يمكن فقط للمستخدمين المدعوين الانضمام." + +msgid "" +"Email used to contact one or all group members, such as a mailing list, " +"shared email, or exchange group." +msgstr "" +"البريد الإلكتروني مستخدم للاتصال بأحد أعضاء المجموعة أو جميعهم، مثل القائمة " +"البريدية أو البريد الإلكتروني المشترك أو مجموعة التبادل." + +msgid "Logo" +msgstr "علامة" + +msgid "Email" +msgstr "بريد إلكتروني" + +msgid "A space or comma-separated list of keywords" +msgstr "مساحة أو قائمة مفصولة بفواصل من الكلمات الرئيسية" + +msgid "Access" +msgstr "دخول" + +msgid "Categories" +msgstr "الفئات" + +msgid "Manager" +msgstr "مدير" + +msgid "Member" +msgstr "عضو" + +msgid "Activity Feed for" +msgstr "خلاصة الأنشطة ل" + +msgid "All" +msgstr "جميع" + +#, fuzzy +#| msgid "dataset" +msgid "Datasets" +msgstr "مجموعة البيانات" + +msgid "Maps" +msgstr "خرائط" + +msgid "Documents" +msgstr "الوثائق" + +msgid "Comments" +msgstr "تعليقات" + +msgid "No actions yet" +msgstr "أية إجراءات بعد" + +msgid "Explore Group Categories" +msgstr "تصفح فئات المجموعة" + +msgid "Create a New Group Category" +msgstr "إنشاء فئة مجموعة جديدة" + +msgid "Explore Groups" +msgstr "تصفح المجموعات" + +msgid "Create a Group" +msgstr "إنشاء مجموعة جديد" + +msgid "Update Group" +msgstr "تحديث المجموعة" + +msgid "Remove Group" +msgstr "إزالة المجموعة" + +msgid "Create Group" +msgstr "إنشاء مجموعة" + +msgid "Last Modified" +msgstr "آخر تعديل" + +msgid "This group has not created a logo." +msgstr "لم تنشئ هذه المجموعة شعارا." + +msgid "Edit Group Details" +msgstr "تحرير تفاصيل المجموعة" + +msgid "Manage Group Members" +msgstr "إدارة أعضاء المجموعة" + +msgid "Delete this Group" +msgstr "حذف هذه المجموعة" + +msgid "Group Activities" +msgstr "نشاطات المجموعة" + +#, python-format +msgid "" +"\n" +" This group is %(access)s.\n" +" " +msgstr "" +"\n" +"\n" +"هذه المجموعة هي%(access)s" + +msgid "Anyone may join this group." +msgstr "يمكن لأي شخص الانضمام إلى هذه المجموعة." + +msgid "Join Group" +msgstr "انضمام مجموعة" + +msgid "Anyone may view this group but membership is by invitation only." +msgstr "يمكن لأي شخص عرض هذه المجموعة ولكن العضوية بالدعوة فقط." + +msgid "Membership is by invitation only." +msgstr "العضوية عن طريق الدعوة فقط." + +msgid "Managers" +msgstr "مدراء" + +msgid "Members" +msgstr "أعضاء" + +msgid "Create a New Group" +msgstr "إنشاء مجموعة جديدة" + +msgid "Edit Members for" +msgstr "تعديل أعضاء ل" + +msgid "Current Members" +msgstr "الأعضاء الحاليين" + +msgid "Remove" +msgstr "إزالة" + +msgid "Promote" +msgstr "ترويج" + +msgid "Demote" +msgstr "تخفيض" + +msgid "Role" +msgstr "وظيفة" + +msgid "Add new members" +msgstr "إضافة أعضاء جدد" + +msgid "Add Group Members" +msgstr "إضافة أعضاء للمجموعة" + +#, python-format +msgid "" +"\n" +" Are you sure you want to remove the group: %(group_title)s?\n" +" " +msgstr "" +"\n" +"\n" +"هل أنت متأكد أنك تريد إزالة المجموعة: %(group_title)s" + +msgid "Yes, I am sure" +msgstr "نعم، أنا متأكد" + +msgid "No, don't remove it" +msgstr "لا، لا تقم بإزالته" + +msgid "Remove Maps/Data from Group" +msgstr "إزالة خرائط / بيانات من المجموعة" + +msgid "Remove selected maps/data" +msgstr "إزالة خرائط اختيار / بيانات" + +msgid "Edit Group Category Details" +msgstr "تعديل تفاصيل فئة المجموعة" + +msgid "Groups" +msgstr "مجموعات" + +msgid "Create Group Category" +msgstr "إنشاء فئة المجموعة" + +msgid "Create a Group Category" +msgstr "إنشاء فئة المجموعة" + +msgid "Edit Group Category" +msgstr "تعديل فئة المجموعة" + +msgid "Save" +msgstr "حفظ" + +msgid "No ready harvesters have been selected, skipping..." +msgstr "" + +msgid "" +"No active refresh sessions have been found for the selected harvesters. " +"Skipping..." +msgstr "" + +msgid "" +"No active harvesting sessions have been found for the selected harvesters. " +"Skipping..." +msgstr "" + +#, python-format +msgid "Resetting status for harvester %(name)s..." +msgstr "" + +#, fuzzy +#| msgid "Harvesting resources..." +msgid "Harvestable resources" +msgstr "حصاد الموارد ..." + +msgid "ready" +msgstr "" + +msgid "updating-harvestable-resources" +msgstr "" + +msgid "aborting-update-harvestable-resources" +msgstr "" + +#, fuzzy +#| msgid "Harvesting resources..." +msgid "harvesting-resources" +msgstr "حصاد الموارد ..." + +#, fuzzy +#| msgid "Harvesting resources..." +msgid "aborting-harvesting-resources" +msgstr "حصاد الموارد ..." + +msgid "checking-availability" +msgstr "" + +#, fuzzy +#| msgid "Harvested" +msgid "Harvester name" +msgstr "حصاد" + +msgid "Base URL of the remote service that is to be harvested" +msgstr "" + +msgid "" +"Whether to periodically schedule this harvester to look for resources on the " +"remote service" +msgstr "" + +msgid "" +"How often (in minutes) should new harvesting sessions be automatically " +"scheduled?" +msgstr "" + +msgid "" +"How often (in minutes) should new refresh sessions be automatically " +"scheduled?" +msgstr "" + +msgid "Whether the remote service is known to be available or not" +msgstr "" + +msgid "" +"How often (in minutes) should the remote service be checked for availability?" +msgstr "" + +msgid "Last time the remote server was checked for availability" +msgstr "" + +msgid "Last time the remote server was checked for harvestable resources" +msgstr "" + +#, fuzzy +#| msgid "version of the cited resource" +msgid "Default owner of harvested resources" +msgstr "نسخة من الموارد المذكورة" + +msgid "" +"Should new resources be harvested automatically without explicit selection?" +msgstr "" + +msgid "" +"Orphan resources are those that have previously been created by means of a " +"harvesting operation but that GeoNode can no longer find on the remote " +"service being harvested. Should these resources be deleted from GeoNode " +"automatically? This also applies to when a harvester configuration is " +"deleted, in which case all of the resources that originated from that " +"harvester are now considered to be orphan." +msgstr "" + +msgid "Date of last update to the harvester configuration." +msgstr "" + +msgid "" +"Harvester class used to perform harvesting sessions. New harvester types can " +"be added by an admin by changing the main GeoNode `settings.py` file" +msgstr "" + +msgid "" +"Configuration specific to each harvester type. Please consult GeoNode " +"documentation on harvesting for more info. This field is mandatory, so at " +"the very least an empty object (i.e. {}) must be supplied." +msgstr "" + +msgid "pending" +msgstr "" + +msgid "on-going" +msgstr "" + +msgid "finished-all-ok" +msgstr "" + +msgid "finished-all-failed" +msgstr "" + +msgid "finished-some-failed" +msgstr "" + +#, fuzzy +#| msgid "Ratings" +msgid "aborting" +msgstr "تصنيفات" + +msgid "aborted" +msgstr "" + +#, fuzzy +#| msgid "Harvesting resources..." +msgid "harvesting" +msgstr "حصاد الموارد ..." + +#, fuzzy +#| msgid "Harvesting resources..." +msgid "discover-harvestable-resources" +msgstr "حصاد الموارد ..." + +msgid "Number of records being processed in this session" +msgstr "" + +#, fuzzy +#| msgid "All resources have already been imported" +msgid "Number of records that have already been processed" +msgstr "تم بالفعل استيراد جميع الموارد" + +msgid "updating-harvestable-resource" +msgstr "" + +#, fuzzy +#| msgid "Harvesting resources..." +msgid "harvesting-resource" +msgstr "حصاد الموارد ..." + +msgid "" +"Identifier that allows referencing the resource on its remote service in a " +"unique fashion. This is usually automatically filled by the harvester " +"worker. The harvester worker needs to know how to either read or generate " +"this from each remote resource in order to be able to compare the " +"availability of resources between consecutive harvesting sessions." +msgstr "" + +msgid "" +"Type of the resource in the remote service. Each harvester worker knows how " +"to fill this field, in accordance with the resources for which harvesting is " +"supported" +msgstr "" + +#, fuzzy, python-brace-format +#| msgid "The e-mail address '%(email)s' has already been invited." +msgid "The e-mail address '{email}' has already been invited." +msgstr "تم بالفعل دعوة عنوان البريد الإلكتروني '%(email)s'." + +#, fuzzy, python-brace-format +#| msgid "The e-mail address '%(email)s' has already accepted an invite." +msgid "The e-mail address '{email}' has already accepted an invite." +msgstr "قبل عنوان البريد الإلكتروني '%(email)s' دعوة." + +#, fuzzy, python-brace-format +#| msgid "An active user is already using the e-mail address '%(email)s'" +msgid "An active user is already using the e-mail address '{email}'" +msgstr "مستخدم نشط يستخدم بالفعل عنوان البريد الإلكتروني '%(email)s'" + +msgid "E-mail" +msgstr "بريد إلكتروني" + +#, python-format +msgid "Invitations successfully sent to '%(email)s'" +msgstr "تم إرسال الدعوات بنجاح إلى '%(email)s'" + +#, python-format +msgid "" +"Sorry, it was not possible to invite '%(email)s' due to the following isse: " +"%(error)s (%(type)s)" +msgstr "" +"عذرًا ، لم يكن من الممكن دعوة '%(email)s' بسبب المشكلة التالية: %(error)s " +"(%(type)s)" + +#, fuzzy +#| msgid "dataset" +msgid "Dataset Created" +msgstr "مجموعة البيانات" + +#, fuzzy +#| msgid "A Layer was created" +msgid "A Dataset was created" +msgstr "‫تم إنشاء طبقة مسبقا" + +#, fuzzy +#| msgid "dataset" +msgid "Dataset Updated" +msgstr "مجموعة البيانات" + +#, fuzzy +#| msgid "A Layer was updated" +msgid "A Dataset was updated" +msgstr "تم تحديث الطبقة مسبقا" + +#, fuzzy +#| msgid "Layer Approved" +msgid "Dataset Approved" +msgstr "طبقة موافق عليها" + +#, fuzzy +#| msgid "A Layer was approved by a Manager" +msgid "A Dataset was approved by a Manager" +msgstr "تمت الموافقة على طبقة من قبل مدير" + +#, fuzzy +#| msgid "Is Published" +msgid "Dataset Published" +msgstr "تم نشره" + +#, fuzzy +#| msgid "A Layer was published" +msgid "A Dataset was published" +msgstr "تم نشر طبقة" + +#, fuzzy +#| msgid "Layer Default Style" +msgid "Dataset Deleted" +msgstr "طبقة النمط الافتراضي" + +#, fuzzy +#| msgid "A Layer was deleted" +msgid "A Dataset was deleted" +msgstr "تم حذف الطبقة مسبقا" + +msgid "YYYYMMDD" +msgstr "YYYYMMDD" + +msgid "YYYYMMDD'T'hhmmss" +msgstr "YYYYMMDD'T'hhmmss" + +msgid "YYYYMMDD'T'hhmmss'Z'" +msgstr "YYYYMMDD'T'hhmmss'Z'" + +msgid "style name" +msgstr "اسم النمط" + +msgid "sld text" +msgstr "نص sld" + +msgid "sld version" +msgstr "إصدار sld" + +msgid "sld url" +msgstr "sld رابط" + +msgid "Workspace" +msgstr "مساحة العمل" + +msgid "Store" +msgstr "مخزن" + +msgid "Typename" +msgstr "أكتب اسم" + +msgid "ows URL" +msgstr "ows URL" + +msgid "The URL of the OWS service providing this layer, if any exists." +msgstr "" + +msgid "Is mosaic?" +msgstr "هل هي فسيفساء؟" + +msgid "Has time?" +msgstr "لديه وقت؟" + +msgid "Has elevation?" +msgstr "لديه ارتفاع؟" + +msgid "Time regex" +msgstr "وقت التعبير العادي" + +msgid "Elevation regex" +msgstr "ارتفاع التعبير العادي" + +#, fuzzy +#| msgid "Type" +msgid "P-Type" +msgstr "نوع" + +msgid "use featureinfo custom template?" +msgstr "" + +msgid "specifies wether or not use a custom GetFeatureInfo template." +msgstr "" + +msgid "featureinfo custom template" +msgstr "" + +msgid "the custom GetFeatureInfo template HTML contents." +msgstr "" + +msgid "File cannot be opened, maybe check the encoding" +msgstr "لا يمكن فتح الملف، ربما تحقق من الترميز" + +msgid "attribute name" +msgstr "اسم السمة" + +msgid "name of attribute as stored in shapefile/spatial database" +msgstr "اسم السمة كما تم تخزينها في shapefile/ قاعدة البيانات المكانية" + +msgid "attribute description" +msgstr "وصف السمة" + +msgid "description of attribute to be used in metadata" +msgstr "وصف السمة التي سيتم استخدامها في البيانات الوصفية" + +msgid "attribute label" +msgstr "تسمية السمة" + +msgid "title of attribute as displayed in GeoNode" +msgstr "عنوان السمة كما هو معروض في GeoNode" + +msgid "attribute type" +msgstr "نوع السمة" + +msgid "the data type of the attribute (integer, string, geometry, etc)" +msgstr "نوع بيانات السمة (عدد صحيح، سلسلة، والهندسة، وغيرها)" + +msgid "visible?" +msgstr "مرئية؟" + +msgid "specifies if the attribute should be displayed in identify results" +msgstr "يحدد ما إذا كان يجب عرض السمة في تحديد النتائج" + +msgid "display order" +msgstr "ترتيب العرض" + +msgid "" +"specifies the order in which attribute should be displayed in identify " +"results" +msgstr "يحدد الترتيب الذي يجب أن يتم عرض السمة في تحديد النتائج" + +msgid "Label" +msgstr "علامة" + +#, fuzzy +#| msgid "Images" +msgid "Image" +msgstr "الصور" + +msgid "Video (mp4)" +msgstr "" + +msgid "Video (ogg)" +msgstr "" + +msgid "Video (webm)" +msgstr "" + +msgid "Video (3gp)" +msgstr "" + +msgid "Video (flv)" +msgstr "" + +msgid "Video (YouTube/VIMEO - embedded)" +msgstr "" + +msgid "Audio" +msgstr "" + +msgid "IFRAME" +msgstr "" + +#, fuzzy +#| msgid "feature type" +msgid "featureinfo type" +msgstr "نوع الميزة" + +#, fuzzy +#| msgid "specifies if the attribute should be displayed in identify results" +msgid "" +"specifies if the attribute should be rendered with an HTML widget on " +"GetFeatureInfo template." +msgstr "يحدد ما إذا كان يجب عرض السمة في تحديد النتائج" + +msgid "count" +msgstr "إحصاء" + +msgid "count value for this field" +msgstr "قيمة العد لهذا الحقل" + +msgid "min" +msgstr "الحد الأدنى" + +msgid "minimum value for this field" +msgstr "قيمة الحد الأدنى لهذا الحقل" + +msgid "max" +msgstr "الحد الأقصى" + +msgid "maximum value for this field" +msgstr "القيمة القصوى لهذا الحقل" + +msgid "average" +msgstr "متوسط" + +msgid "average value for this field" +msgstr "متوسط ​قيمة لهذا الحقل" + +msgid "median" +msgstr "الوسيط" + +msgid "median value for this field" +msgstr "متوسط ​​قيمة لهذا الحقل" + +msgid "standard deviation" +msgstr "الانحراف المعياري" + +msgid "standard deviation for this field" +msgstr "الانحراف المعياري لهذا الحقل" + +msgid "sum" +msgstr "مجموع" + +msgid "sum value for this field" +msgstr "قيمة المجموع لهذا الحقل" + +msgid "unique values for this field" +msgstr "قيم فريدة لهذا الحقل" + +msgid "last modified" +msgstr "آخر تعديل" + +msgid "date when attribute statistics were last updated" +msgstr "التاريخ الذي تم فيه تحديث آخر إحصاءات السمات" + +#, fuzzy +#| msgid "Return to Layer" +msgid "Append to Dataset" +msgstr "العودة إلى الطبقة" + +#, fuzzy +#| msgid "Explore Layers" +msgid "Explore Datasets" +msgstr "تصفح الطبقات" + +#, fuzzy +#| msgid "Return to Layer" +msgid "Return to Dataset" +msgstr "العودة إلى الطبقة" + +msgid "Preserve Metadata XML" +msgstr "الحفاظ على البيانات الوصفية لXML" + +msgid "Select the charset or leave default" +msgstr "حدد محارف أو اترك الإفتراضي" + +#, fuzzy +#| msgid "Layer WMS GetCapabilities document" +msgid "Dataset WMS GetCapabilities document" +msgstr "احصل على مستند القدرات لطبقات المستخدم لبرنامج إدارة المستودعات" + +msgid "Filter Granules" +msgstr "تصفية الحبيبات" + +msgid "Active Filter:" +msgstr "تصفية نشطة:" + +msgid "Granule ID" +msgstr "معرف الحبيبية" + +msgid "Bounding Box" +msgstr "المربع المحيط" + +msgid "Time" +msgstr "زمن" + +msgid "Actions" +msgstr "أفعال" + +msgid "Granule Preview" +msgstr "معاينة الحبيبية" + +msgid "Granule Remove" +msgstr "إزالةالحبيبية" + +msgid "Dimension" +msgstr "البعد" + +msgid "Enabled" +msgstr "تمكين" + +msgid "Regex" +msgstr "تعبير منتظم" + +msgid "TIME" +msgstr "زمن" + +msgid "ELEVATION" +msgstr "ارتفاع" + +msgid "Attribute Name" +msgstr "اسم السمة" + +msgid "Range" +msgstr "نطاق" + +msgid "Average" +msgstr "متوسط" + +msgid "Median" +msgstr "الوسيط" + +msgid "Standard Deviation" +msgstr "الانحراف المعياري" + +#, fuzzy +#| msgid "Rate this layer" +msgid "Rate this dataset" +msgstr "قيم هذه الطبقة" + +msgid "Average Rating" +msgstr "متوسط ​​التقييم" + +msgid "Analyze with" +msgstr "تحليل ب" + +#, fuzzy +#| msgid "Download the" +msgid "Download Dataset" +msgstr "تحميل" + +msgid "Request Download" +msgstr "طلب تحميل" + +msgid "Images" +msgstr "الصور" + +msgid "Data" +msgstr "البيانات" + +#, fuzzy +#| msgid "Click to filter the layer" +msgid "Click to filter the dataset" +msgstr "انقر لتصفية الطبقة" + +msgid "Do you want to filter it?" +msgstr "هل تريد تصفية ذلك؟" + +msgid "Loading..." +msgstr "جار التحميل..." + +msgid "Filter by attributes" +msgstr "تصفية حسب السمات" + +msgid "Match" +msgstr "مباراة" + +msgid "all" +msgstr "جميع" + +msgid "any" +msgstr "أي" + +msgid "of the following:" +msgstr "الخريطة التالية:" + +msgid "Attribute" +msgstr "سمة" + +msgid "Operator" +msgstr "المشغل" + +msgid "Feature limit" +msgstr "حد الميزة" + +msgid "Pick your download format:" +msgstr "اختر تنسيق التنزيل الخاص بك:" + +msgid "" +"No data available for this resource. Please contact a system administrator " +"or a manager." +msgstr "" +"لا توجد بيانات متوفرة لهذا المورد. الرجاء الاتصال بمسؤول النظام أو مدير." + +msgid "Warning" +msgstr "تحذير" + +msgid "Close" +msgstr "أغلق" + +msgid "Metadata Detail" +msgstr "تفاصيل البيانات الوصفية" + +#, fuzzy +#| msgid "Request Download" +msgid "Request change" +msgstr "طلب تحميل" + +msgid "Editing Tools" +msgstr "أدوات التعديل" + +#, fuzzy +#| msgid "Edit data" +msgid "Edit dataset" +msgstr "تعديل البيانات" + +msgid "Wizard" +msgstr "معالج" + +msgid "Advanced Edit" +msgstr "تعديل متقدم" + +msgid "Upload Metadata" +msgstr "تحميل البيانات الوصفية" + +msgid "Styles" +msgstr "الأنماط" + +msgid "Set" +msgstr "وضع" + +msgid "Clone" +msgstr "" + +msgid "Replace" +msgstr "استبدال" + +msgid "Append Data" +msgstr "" + +msgid "Download Metadata" +msgstr "تحميل البيانات الوصفية" + +msgid "Full metadata" +msgstr "بيانات وصفيه كامله" + +msgid "Text format" +msgstr "تنسيق النص" + +msgid "HTML format" +msgstr "تنسيق HTML" + +msgid "Standard Metadata - XML format" +msgstr "البيانات الوصفية القياسية-تنسيق XML" + +msgid "Attribute Information" +msgstr "معلومات السمة" + +msgid "ISO Feature Catalogue" +msgstr "كتالوج ميزة ISO" + +msgid "Legend" +msgstr "مفتاح رسم بياني" + +#, fuzzy +#| msgid "Maps using this layer" +msgid "Maps using this dataset" +msgstr "خرائط باستخدام هذه الطبقة" + +#, fuzzy +#| msgid "List of maps using this layer:" +msgid "List of maps using this dataset:" +msgstr "قائمة الخرائط باستخدام هذه الطبقة:" + +#, fuzzy +#| msgid "This layer is not currently used in any maps." +msgid "This dataset is not currently used in any maps." +msgstr "لا تستخدم هذه الطبقة في الوقت الراهن في أي خرائط." + +#, fuzzy +#| msgid "Documents related to this layer" +msgid "Documents related to this dataset" +msgstr "الوثائق المتصلة بهذه الطبقة" + +#, fuzzy +#| msgid "List of documents related to this layer:" +msgid "List of documents related to this dataset:" +msgstr "قائمة الوثائق ذات الصلة لهذه الطبقة:" + +#, fuzzy +#| msgid "" +#| "The following styles are associated with this layer. Choose a style to " +#| "view it in the preview map." +msgid "" +"The following styles are associated with this dataset. Choose a style to " +"view it in the preview map." +msgstr "" +"ترتبط الأنماط التالية مع هذه الطبقة. اختر نمط لمشاهدته في معاينة الخريطة." + +msgid "(default style)" +msgstr "(أسلوب إفتراضي)" + +#, fuzzy +#| msgid "No styles associated with this layer" +msgid "No styles associated with this dataset" +msgstr "لا توجد أنماط مرتبطة بهذه الطبقة" + +#, fuzzy +#| msgid "External service layer" +msgid "External service dataset" +msgstr "طبقة الخدمة الخارجية" + +msgid "Source" +msgstr "مصدر" + +#, fuzzy +#| msgid "Refresh Attributes and Statistics of this layer" +msgid "Refresh Attributes and Statistics of this dataset" +msgstr "تحديث سمات وإحصائيات هذه الطبقة" + +#, fuzzy +#| msgid "" +#| "Click the button below to allow GeoNode refreshing the list of available " +#| "Layer Attributes. If the option 'WPS_ENABLED' has been also set on the " +#| "backend, it will recalculate their statistics too." +msgid "" +"Click the button below to allow GeoNode refreshing the list of available " +"Dataset Attributes. If the option 'WPS_ENABLED' has been also set on the " +"backend, it will recalculate their statistics too." +msgstr "" +"انقر فوق الزر أدناه للسماح لـ GeoNode بتحديث قائمة سمات الطبقة المتاحة. إذا " +"تم تعيين الخيار \"WPS_ENABLED\" أيضًا على الواجهة الخلفية ، فسيعيد حساب " +"إحصائياتهم أيضًا." + +msgid "Refresh Attributes and Statistics" +msgstr "تحديث السمات والإحصائيات" + +#, fuzzy +#| msgid "Clear the Server Cache of this layer" +msgid "Clear the Server Cache of this dataset" +msgstr "امسح ذاكرة التخزين المؤقتة للخادم في هذه الطبقة" + +#, fuzzy +#| msgid "Click the button below to wipe the tile-cache of this layer." +msgid "Click the button below to wipe the tile-cache of this dataset." +msgstr "انقر فوق الزر أدناه لمسح ذاكرة التخزين المؤقت للبلاط لهذه الطبقة." + +#, fuzzy +#| msgid "Empty Tiled-Layer Cache" +msgid "Empty Tiled-Dataset Cache" +msgstr "ذاكرة التخزين المؤقت لطبقةالبلاط فارغة" + +#, fuzzy +#| msgid "Click the button below to change the permissions of this layer." +msgid "Click the button below to change the permissions of this dataset." +msgstr "انقر فوق الزر أدناه لتغيير الأذونات من هذه الطبقة." + +#, fuzzy +#| msgid "Change Layer Permissions" +msgid "Change Dataset Permissions" +msgstr "غير صلاحيات طبقة" + +msgid "Remove Mosaic Granules" +msgstr "إزالة حبيبات الفسيفساء" + +#, fuzzy, python-format +#| msgid "" +#| "\n" +#| " Are you sure you want to remove Granule %(granule_id)s of the " +#| "Mosaic %(layer_title)s?\n" +#| " " +msgid "" +"\n" +" Are you sure you want to remove Granule %(granule_id)s of the " +"Mosaic %(dataset_title)s?\n" +" " +msgstr "" +"\n" +"هل تريد بالتأكيد إزالة الحبيبة %(granule_id)s من الفسيفساء%(layer_title)s?" + +msgid "This action affects the following maps:" +msgstr "هذا العمل يؤثر على الخرائط التالية:" + +#, fuzzy +#| msgid "No maps are using this layer" +msgid "No maps are using this dataset" +msgstr "لا خرائط تستخدم هذه الطبقة" + +#, fuzzy, python-format +#| msgid "for %(layer_title)s" +msgid "for %(dataset_title)s" +msgstr "لـ%(layer_title)s" + +msgid "" +"Note: this layer's orginal metadata was populated and preserved by importing " +"a metadata XML file.\n" +" This metadata cannot be edited." +msgstr "" +"ملاحظة: تم تعبئة بيانات التعريف الأصلية لهذه الطبقة وحفظها عن طريق استيراد " +"ملف XML لبيانات التعريف. لا يمكن تعديل هذه البيانات التعريفية." + +msgid "" +"Note: this layer's orginal metadata was populated by importing a metadata " +"XML file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and " +"Dublin Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" +"ملاحظة: تم ملء بيانات التعريف الأولية لهذه الطبقة باستيراد ملف XML شمل " +"للبيانات الوصفية.\n" +"‬\n" +"‫           يدعم استيراد البيانات الوصفية ل GeoNode مجموعة فرعية من عناصر " +"البيانات الوصفية ISO, FGDC, and Dublin Core.\n" +"‬\n" +"‫          ربما فقدت بعض البيانات الوصفية الأصلية." + +#, fuzzy, python-format +#| msgid "" +#| "\n" +#| " Editing details for %(layer_title)s\n" +#| " " +msgid "" +"\n" +" Editing details for %(dataset_title)s\n" +" " +msgstr "" +"‫\n" +"‬\n" +"‫‫‫‫‫تعديل التفاصيل ل%(layer_title)s " + +msgid "Use a custom template?" +msgstr "" + +msgid "Display Order" +msgstr "ترتيب العرض" + +#, fuzzy +#| msgid "Display Order" +msgid "Display Type" +msgstr "ترتيب العرض" + +msgid "Visible" +msgstr "مرئية" + +msgid "ERROR" +msgstr "خطأ" + +msgid "Following items are required:" +msgstr "" + +#, fuzzy +#| msgid "Upload Layer Metadata" +msgid "Upload Dataset Metadata" +msgstr "تحميل البيانات الوصفية للطبقة" + +msgid "(XML - ISO, FGDC, ebRIM, Dublin Core)" +msgstr "(XML - ISO, FGDC, ebRIM, Dublin Core)" + +msgid "Click the button below to change the permissions of this layer." +msgstr "انقر فوق الزر أدناه لتغيير الأذونات من هذه الطبقة." + +#, fuzzy +#| msgid "Replace Layer" +msgid "Replace Dataset" +msgstr "استبدال طبقة" + +msgid "Manage Styles" +msgstr "إدارة أنماط" + +#, fuzzy, python-format +#| msgid "" +#| "\n" +#| " Manage Available Styles for " +#| "%(layer_title)s\n" +#| " " +msgid "" +"\n" +" Manage Available Styles for " +"%(dataset_title)s\n" +" " +msgstr "" +"‫\n" +"‬\n" +"‫‫إدارة الأنماط المتاحة ل %(layer_title)s " + +#, fuzzy +#| msgid "Layer Default Style" +msgid "Dataset Default Style" +msgstr "طبقة النمط الافتراضي" + +msgid "Available styles" +msgstr "الأساليب المتاحة" + +msgid "" +"Click on an available style in the upper box to assign it to this layer. " +"Selected styles appear in the lower box." +msgstr "" +"انقر على نمط متاح في المربع العلوي لتعيينه لهذه الطبقة.تظهر الأنماط المحددة " +"في المربع السفلي." + +msgid "Update Available Styles" +msgstr "تحديث الأنماط المتوفرة" + +#, fuzzy +#| msgid "Upload Layer Style" +msgid "Upload Dataset Style" +msgstr "تحميل نمط الطبقة" + +msgid "(SLD - Style Layer Descriptor 1.0, 1.1)" +msgstr "(SLD - واصف نمط طبقة 1.0 ، 1.1)" + +msgid "WARNING" +msgstr "تحذير" + +msgid "This will most probably overwrite the current default style!" +msgstr "سيؤدي هذا على الأرجح إلى الكتابة فوق النمط الافتراضي الحالي!" + +msgid "Preview" +msgstr "معاينة" + +msgid "Dataset Attributes" +msgstr "سمات مجموعة البيانات" + +#, fuzzy, python-brace-format +#| msgid "You are attempting to replace a vector layer with a raster." +msgid "You are attempting to {action_type} a vector dataset with a raster." +msgstr "أنت تحاول استبدال طبقة متجه بنقطية." + +#, fuzzy, python-brace-format +#| msgid "You are attempting to replace a raster layer with a vector." +msgid "You are attempting to {action_type} a raster dataset with a vector." +msgstr "أنت تحاول استبدال طبقة النقطية بناقلات." + +#, fuzzy, python-brace-format +#| msgid "You are attempting to replace a vector layer with a raster." +msgid "" +"You are attempting to {action_type} a vector dataset with an unknown format." +msgstr "أنت تحاول استبدال طبقة متجه بنقطية." + +#, fuzzy, python-brace-format +#| msgid "" +#| "Please ensure the name is consistent with the file you are trying to " +#| "replace." +msgid "" +"Please ensure the name is consistent with the file you are trying to " +"{action_type}." +msgstr "الرجاء التأكد من أن الاسم متوافق مع الملف الذي تحاول استبداله." + +#, fuzzy +#| msgid "Local GeoNode layer has no geometry type." +msgid "Local GeoNode dataset has no geometry type." +msgstr "طبقة GeoNode المحلية ليس لها نوع الهندسة." + +#, fuzzy, python-brace-format +#| msgid "" +#| "Please ensure there is at least one geometry " +#| "type that is consistent with the file you " +#| "are trying to replace." +msgid "" +"Please ensure there is at least one geometry " +"type that is consistent with the file you are " +"trying to {action_type}." +msgstr "" +"الرجاء التأكد من وجود نوع هندسة واحد على الأقل يتسق مع الملف الذي تحاول " +"استبداله." + +msgid "The selected Dataset does not exists in the catalog." +msgstr "" + +msgid "Please ensure that the dataset structure is consistent " +msgstr "" + +#, fuzzy +#| msgid "Some error occurred while trying to access the uploaded schema: %s" +msgid "" +"Some error occurred while trying to access the uploaded schema: {str(e)}" +msgstr "حدث بعض الأخطاء أثناء محاولة الوصول إلى المخطط الذي تم تحميله: %s" + +msgid "" +"There was an error while attempting to upload your data. Please try again, " +"or contact and administrator if the problem continues." +msgstr "" +"حدث خطأ أثناء محاولة تحميل بياناتك. الرجاء إعادة المحاولة، أو الاتصال " +"بالمشرف إذا استمرت المشكلة." + +#, fuzzy +#| msgid "" +#| "Note: this layer's orginal metadata was populated and preserved by " +#| "importing a metadata XML file. This metadata cannot be edited." +msgid "" +"Note: this dataset's orginal metadata was populated and preserved by " +"importing a metadata XML file. This metadata cannot be edited." +msgstr "" +"ملاحظة: تم تعبئة بيانات التعريف الأصلية لهذه الطبقة وحفظها عن طريق استيراد " +"ملف XML لبيانات التعريف. لا يمكن تعديل هذه البيانات التعريفية." + +#, fuzzy +#| msgid "You are not permitted to delete this layer" +msgid "You are not permitted to delete this dataset" +msgstr "لا يسمح لك بحذف هذه الطبقة" + +#, fuzzy +#| msgid "You do not have permissions for this layer." +msgid "You do not have permissions for this dataset." +msgstr "لم يكن لديك صلاحيات لهذه الطبقة." + +#, fuzzy +#| msgid "You are not permitted to modify this layer" +msgid "You are not permitted to modify this dataset" +msgstr "لا يسمح لك بتعديل هذه الطبقة" + +#, fuzzy +#| msgid "You are not permitted to modify this layer's metadata" +msgid "You are not permitted to modify this dataset's metadata" +msgstr "لا يسمح لك بتعديل البيانات الوصفية لهذه الطبقة" + +#, fuzzy +#| msgid "You are not permitted to view this layer" +msgid "You are not permitted to view this dataset" +msgstr "لا يسمح لك بمشاهدة هذه الطبقة" + +#, fuzzy +#| msgid "" +#| "This layer is a member of a layer group, you must remove the layer from " +#| "the group before deleting." +msgid "" +"This dataset is a member of a layer group, you must remove the dataset from " +"the group before deleting." +msgstr "" +"هذه الطبقة عضو في مجموعة الطبقات، يجب إزالة الطبقة من المجموعة قبل الحذف." + +#, fuzzy +#| msgid "Position Name" +msgid "Positional Arguments" +msgstr "اسم المركز" + +msgid "JSON encoded positional arguments (Example: [\"arg1\", \"arg2\"])" +msgstr "" + +#, fuzzy +#| msgid "Keywords" +msgid "Keyword Arguments" +msgstr "الكلمات الدالة" + +msgid "JSON encoded keyword arguments (Example: {\"argument\": \"value\"})" +msgstr "" + +#, python-brace-format +msgid "" +"You can not start the job {job.id} because its status is \"{job.status}\"." +msgstr "" + +msgid "Map Created" +msgstr "تم إنشاء الخريطة" + +msgid "A Map was created" +msgstr "تم إنشاء خريطة" + +msgid "Map Updated" +msgstr "تم تحديث الخريطة" + +msgid "A Map was updated" +msgstr "تم تحديث الخريطة مسبقا" + +msgid "Map Approved" +msgstr "خريطة موافق عليها" + +msgid "A Map was approved by a Manager" +msgstr "تمت الموافقة على الخريطة من قبل المدير" + +msgid "Map Published" +msgstr "الخريطة نشرت" + +msgid "A Map was published" +msgstr "تم نشر خريطة" + +msgid "Map Deleted" +msgstr "تم حذف الخريطة" + +msgid "A Map was deleted" +msgstr "تم حذف الخريطة مسبقا" + +msgid "Comment on Map" +msgstr "التعليق على خريطة" + +msgid "A map was commented on" +msgstr "وقد علق على خريطة" + +msgid "Rating for Map" +msgstr "تصنيف للخريطة" + +msgid "A rating was given to a map" +msgstr "وقدم تصنيف للخريطة" + +msgid "You are not permitted to delete this map." +msgstr "لا يسمح لك بحذف هذه الخريطة." + +msgid "You do not have permissions for this map." +msgstr "ليس لديك صلاحيات لهذه الخريطة." + +msgid "You must be logged in to save this map" +msgstr "يجب أن تكون مسجلا لحفظ هذه الخريطة" + +msgid "You are not permitted to save or edit this map." +msgstr "لا يسمح لك بحفظ أو تعديل هذه الخريطة." + +msgid "You are not allowed to modify this map's metadata." +msgstr "لا يسمح لك بتعديل البيانات الوصفية هذه الخريطة ل." + +msgid "You are not allowed to view this map." +msgstr "لا يسمح لك بعرض هذه الخريطة." + +msgid "Site URL" +msgstr "رابط الموقع" + +msgid "Featured Map URL" +msgstr "رابط خريطة مميزة" + +msgid "name" +msgstr "اسم" + +#, fuzzy +#| msgid "Store" +msgid "store" +msgstr "مخزن" + +msgid "current style" +msgstr "" + +msgid "" +"\n" +" Here you can download all the layers of this map that\n" +" are hosted on this GeoNode.\n" +" " +msgstr "" +"\n" +"‫‫هنا يمكنك تحميل جميع طبقات هذه الخريطة المستضافة على GeoNode." + +msgid "" +"\n" +"
          Could not find downloadable layers " +"for this map. You can go back to \n" +" " +msgstr "" +"\n" +"
          تعذر العثور على طبقات قابلة للتنزيل لهذه " +"الخريطة. يمكنك العودة إلى " + +msgid "" +"\n" +" Additionally, the map contains these layers which will not be " +"downloaded\n" +" due to security restrictions:\n" +" " +msgstr "" +"\n" +"بالإضافة إلى ذلك، تحتوي الخريطة على هذه الطبقات التي لن يتم تنزيلها\n" +"‫‫بسبب القيود الأمنية:" + +msgid "" +"\n" +" Finally, the map contains these layers which will not be downloaded\n" +" because they are not available directly from this GeoNode:\n" +" " +msgstr "" +"\n" +"أخيرا، تحتوي الخريطة على هذه الطبقات التي لن يتم تنزيلها\n" +"‫‫ لأنها غير متوفرة مباشرة من هذا GeoNode:" + +msgid "Downloading" +msgstr "تحميل" + +msgid "Retry" +msgstr "إعادة المحاولة" + +msgid "Start downloading this map" +msgstr "بدء تنزيل هذه الخريطة" + +msgid "Download Complete" +msgstr "تحميل منتهي" + +msgid "Download Failed" +msgstr "فشل تحميل" + +msgid "" +"Note: this map's orginal metadata was populated by importing a metadata XML " +"file.\n" +" GeoNode's metadata import supports a subset of ISO, FGDC, and Dublin " +"Core metadata elements.\n" +" Some of your original metadata may have been lost." +msgstr "" +"ملاحظة: تم ملء البيانات الوصفية الأصلية لهذه الخريطة عن طريق استيراد ملف XML " +"شامل للبيانات الوصفية.\n" +"‬\n" +"‫         يدعم استيراد البيانات الوصفية ل GeoNode مجموعة فرعية من عناصر " +"البيانات الوصفية ISO, FGDC, and Dublin Core.\n" +"‬\n" +"‫         ربما فقدت بعض البيانات الوصفية الأصلية." + +msgid "Explore Maps" +msgstr "تصفح الخرائط" + +msgid "Return to Map" +msgstr "العودة إلى الخريطة" + +#, fuzzy +#| msgid "Message User" +msgid "Message received" +msgstr "رسالة المستخدم" + +msgid "New message received in one of your threads" +msgstr "رسالة جديدة تم تلقيها في أحد مؤشرات الترابط الخاصة بك" + +#, fuzzy +#| msgid "Monitoring" +msgid "Monitoring alert" +msgstr "مراقبة" + +msgid "Alert situation reported by monitoring" +msgstr "حالة التأهب المبلغ عنها بواسطة المراقبة" + +#, fuzzy +#| msgid "GeoNode Monitoring on {} reports errors: {}" +msgid "GeoNode Monitoring on {base_ctx['host']} " +msgstr "رصد GeoNode {} تقارير الأخطاء: {}" + +msgid "Enter valid email addresses." +msgstr "تحقق من عنوان بريدك الإلكتروني." + +msgid "Show list of services" +msgstr "عرض قائمة الخدمات" + +msgid "" +"Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, last sync will be used." +msgstr "" +"معالجة البيانات منذ الطابع الزمني المحدد (YYYY-MM-DD HH: MM: SS التنسيق). " +"إذا لم يتم توفير ذلك ، فسيتم استخدام المزامنة الأخيرة." + +msgid "" +"Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). If not " +"provided, now will be used." +msgstr "" +"معالجة البيانات حتى الطابع الزمني المحدد (YYYY-MM-DD HH: MM: SS التنسيق). " +"إذا لم تقدم ، سيتم استخدامها الآن." + +msgid "Force check" +msgstr "تحقق القوة" + +msgid "Format of audit log (xml, json)" +msgstr "تنسيق سجل التدقيق (xml ، json)" + +msgid "" +"Should old data be preserved (default: no, data older than settings." +"MONITORING_DATA_TTL will be removed)" +msgstr "" +"في حالة الاحتفاظ بالبيانات القديمة (افتراضي: لا ، ستتم إزالة البيانات الأقدم " +"من الإعدادات. ستتم إزالة MONITORING_DATA_TTL)" + +msgid "Should stop on first error occured (default: no)" +msgstr "يجب التوقف عند حدوث الخطأ الأول (الافتراضي: لا)" + +msgid "Should process and send notifications as well (default: no)" +msgstr "يجب معالجة وإرسال الإشعارات أيضًا (الافتراضي: لا)" + +msgid "Collect data from this service only" +msgstr "جمع البيانات من هذه الخدمة فقط" + +msgid "Show list of metrics" +msgstr "عرض قائمة المقاييس" + +msgid "Show list of labels for metric" +msgstr "عرض قائمة التسميات للقياس المتري" + +msgid "Show list of resources for metric" +msgstr "عرض قائمة الموارد للقياس" + +msgid "Data aggregation interval in seconds (default: 60)" +msgstr "الفاصل الزمني لتجميع البيانات بالثواني (الافتراضي: 60)" + +msgid "Metric name" +msgstr "اسم قياسي" + +msgid "Show data for specific resource in resource_type=resource_name format" +msgstr "إظهار البيانات لمورد معين في resource_type = resource_name format" + +msgid "Show data for specific resource" +msgstr "عرض البيانات لمورد معين" + +msgid "Show data for specific label" +msgstr "عرض البيانات لتسمية محددة" + +#, fuzzy, python-brace-format +#| msgid "Write result to file, default GEOIP_PATH: {}" +msgid "Write result to file, default GEOIP_PATH: {settings.GEOIP_PATH}" +msgstr "اكتب نتيجة الملف ، الافتراضي GEOIP_PATH: {}" + +msgid "" +"Fetch database from specific url. If nothing provided, default {} will be " +"used" +msgstr "" +"إحضار قاعدة البيانات من رابط معين. إذا لم يتم توفير أي شيء ، فسيتم استخدام " +"{} الافتراضي" + +msgid "Overwrite file if exists" +msgstr "الكتابة فوق الملف إذا كان موجودًا" + +msgid "GeoNode" +msgstr "GeoNode" + +msgid "GeoServer" +msgstr "GeoServer" + +msgid "Host (GeoServer)" +msgstr "المضيف (GeoServer)" + +msgid "Host (GeoNode)" +msgstr "المضيف (GeoNode)" + +msgid "No resource" +msgstr "لا مورد" + +#, fuzzy +#| msgid "dataset" +msgid "Dataset" +msgstr "مجموعة البيانات" + +msgid "Map" +msgid_plural "Maps" +msgstr[0] "خريطة" +msgstr[1] "خريطة" +msgstr[2] "خريطتين" +msgstr[3] "خرائط" +msgstr[4] "خرائط" +msgstr[5] "خرائط" + +#, fuzzy +#| msgid "Resources" +msgid "Resource base" +msgstr "موارد" + +msgid "Document" +msgid_plural "Documents" +msgstr[0] "مستند" +msgstr[1] "مستند" +msgstr[2] "مستندان/مستندين" +msgstr[3] "مستندات" +msgstr[4] "مستندات" +msgstr[5] "مستندات" + +msgid "Style" +msgstr "نمط" + +msgid "Admin" +msgstr "المشرف" + +msgid "Other" +msgstr "آخر" + +msgid "Rate" +msgstr "صنف" + +msgid "Count" +msgstr "إحصاء" + +msgid "Value" +msgstr "القيمة" + +msgid "Value numeric" +msgstr "قيمة رقمية" + +msgid "Bytes" +msgstr "بايت" + +msgid "Kilobytes" +msgstr "كيلو بايت" + +msgid "Megabytes" +msgstr "ميغابايت" + +msgid "Gigabytes" +msgstr "غيغا بايت" + +msgid "Bytes per second" +msgstr "بايت في الثانية الواحدة" + +msgid "Kilobytes per second" +msgstr "كيلو بايت في الثانية الواحدة" + +msgid "Megabytes per second" +msgstr "ميغابايت في الثانية الواحدة" + +msgid "Gigabytes per second" +msgstr "غيغا بايت في الثانية" + +msgid "Seconds" +msgstr "ثواني" + +msgid "Percentage" +msgstr "النسبة المئوية" + +#, fuzzy +#| msgid "local OWS" +msgid "Not OWS" +msgstr "OWS المحلية" + +msgid "Any OWS" +msgstr "أي OWS" + +msgid "Change" +msgstr "تغيير" + +#, fuzzy +#| msgid "Metadata" +msgid "Change Metadata" +msgstr "البيانات الوصفية" + +#, fuzzy +#| msgid "Published" +msgid "Publish" +msgstr "نشرت" + +#, fuzzy +#| msgid "GeoServer" +msgid "Geoserver event" +msgstr "GeoServer" + +msgid "List of resources affected" +msgstr "قائمة الموارد المتأثرة" + +msgid "Response processing time in ms" +msgstr "زمن معالجة الاستجابة بالميلي ثانية" + +msgid "1 minute" +msgstr "1 دقيقة" + +msgid "5 minutes" +msgstr "5 دقائق" + +msgid "10 minutes" +msgstr "10 دقائق" + +msgid "30 minutes" +msgstr "30 دقيقة" + +msgid "1 hour" +msgstr "ساعة واحدة" + +msgid "Error" +msgstr "قاتلة - مهلك" + +msgid "Fatal" +msgstr "قاتلة - مهلك" + +msgid "Expected min/max values for user configuration" +msgstr "قيم الحد الأدنى / الحد الأقصى المتوقعة لتكوين المستخدم" + +msgid "Marker of last delivery" +msgstr "علامة التسليم الأخير" + +msgid "Minimum time between subsequent notifications" +msgstr "الحد الأدنى من الوقت بين الإشعارات اللاحقة" + +msgid "How severe would be error from this notification" +msgstr "ما مدى خطورة الخطأ من هذا الإشعار" + +msgid "Is it active" +msgstr "هل هو نشط" + +#, python-brace-format +msgid "{self.severity}: {self.name}" +msgstr "" + +msgid "Value must be above" +msgstr "يجب أن تكون القيمة أعلاه" + +msgid "Value must be below" +msgstr "يجب أن تكون القيمة أدناه" + +msgid "Last update must not be older than" +msgstr "يجب ألا يكون التحديث الأخير أقدم من" + +msgid "Max timeout for given metric before error should be raised" +msgstr "الحد الأقصى لمهلة القياس المحددة قبل الخطأ" + +#, fuzzy +#| msgid "Monitoring alert" +msgid "Monitoring & Analytics" +msgstr "تنبيه المراقبة" + +msgid "Only superusers can monitor performance" +msgstr "يمكن للمستخدمين المتميزين فقط مراقبه الأداء" + +msgid "Hi," +msgstr "مرحبا،" + +msgid "Monitoring on" +msgstr "مراقبة على" + +msgid "has detected following problems :" +msgstr "لقد اكتشف المشكلات التالية:" + +msgid "spotted" +msgstr "مراقب" + +msgid "ago" +msgstr "منذ" + +msgid "emerged" +msgstr "ظهرت" + +msgid "User following you" +msgstr "مستخدم يتابعك" + +msgid "Another user has started following you" +msgstr "لقد بدأ مستخدم آخر متابعتك" + +msgid "User requested access" +msgstr "طلب الدخول للمستخدم" + +msgid "A new user has requested access to the site" +msgstr "لقد طلب مستخدم جديد الدخول إلى الموقع" + +msgid "Account activated" +msgstr "تم تنشيط الحساب" + +msgid "This account is now active and can log in the site" +msgstr "هذا الحساب نشط الآن ويمكنه تسجيل الدخول إلى الموقع" + +msgid "Personal info" +msgstr "المعلومات الشخصية" + +msgid "Important dates" +msgstr "التواريخ الهامة" + +msgid "Extended profile" +msgstr "ملف شخصي موسع" + +msgid "Password changed successfully." +msgstr "تم تغيير كلمة المرور بنجاح." + +#, python-format +msgid "Change password: %s" +msgstr "تغيير كلمة المرور: %s" + +msgid "party who authored the resource" +msgstr "الطرف الذي قام بتأليف المورد" + +msgid "" +"party who has processed the data in a manner such that the resource has been " +"modified" +msgstr "الطرف الذي قام بمعالجة البيانات بحيث انه تم تعديل المورد" + +msgid "party who published the resource" +msgstr "الطرف الذي نشر الموارد" + +msgid "" +"party that accepts accountability and responsibility for the data and " +"ensures appropriate care and maintenance of the resource" +msgstr "" +"الطرف الذي يقبل المساءلة والمسؤولية عن البيانات ويضمن الرعاية المناسبة " +"وصيانة المورد" + +msgid "" +"party who can be contacted for acquiring knowledge about or acquisition of " +"the resource" +msgstr "الطرف الذي يمكن الاتصال به للحصول على المعرفة أو الاستحواذ على الموارد" + +msgid "party who distributes the resource" +msgstr "الطرف الذي يوزع الموارد" + +msgid "party who uses the resource" +msgstr "الطرف الذي يستخدم الموارد" + +msgid "party that supplies the resource" +msgstr "الطرف الذي يزود المورد" + +msgid "party who created the resource" +msgstr "الطرف الذي أنشأ المورد" + +msgid "party that owns the resource" +msgstr "الطرف الذي يملك المورد" + +msgid "key party responsible for gathering information and conducting research" +msgstr "الطرف الرئيسي المسؤول عن جمع المعلومات وإجراء البحوث" + +msgid "Email Address" +msgstr "عنوان البريد الإلكتروني" + +msgid "Organization Name" +msgstr "اسم المؤسسة" + +msgid "name of the responsible organization" +msgstr "اسم المنظمة المسؤولة" + +msgid "Profile" +msgstr "الملف الشخصي" + +msgid "introduce yourself" +msgstr "قدم نفسك" + +msgid "Position Name" +msgstr "اسم المركز" + +msgid "role or position of the responsible person" +msgstr "دور أو موقف الشخص المسؤول" + +msgid "Voice" +msgstr "صوت" + +msgid "" +"telephone number by which individuals can speak to the responsible " +"organization or individual" +msgstr "رقم الهاتف الذي يمكن للأفراد التحدث إلى المنظمة المسؤولة أو فرد" + +msgid "Facsimile" +msgstr "الفاكس" + +msgid "" +"telephone number of a facsimile machine for the responsible organization or " +"individual" +msgstr "رقم الهاتف لآلة الفاكس للمنظمة المسؤولة أو الفرد" + +msgid "Delivery Point" +msgstr "نقطة التسليم" + +msgid "" +"physical and email address at which the organization or individual may be " +"contacted" +msgstr "" +"العنوان المادي والبريد الإلكتروني الذي يمكن الاتصال به بالمنظمة أو بالفرد" + +msgid "City" +msgstr "مدينة" + +msgid "city of the location" +msgstr "مدينة الموقع" + +msgid "Administrative Area" +msgstr "منطقة إدارية" + +msgid "state, province of the location" +msgstr "الدولة، محافظة الموقع" + +msgid "Postal Code" +msgstr "الرمز البريدي" + +msgid "ZIP or other postal code" +msgstr "‫ZIP أو رمز بريدي اخر" + +msgid "Country" +msgstr "بلد" + +msgid "country of the physical address" +msgstr "بلد العنوان الفعلي" + +msgid "" +"commonly used word(s) or formalised word(s) or phrase(s) used to describe " +"the subject (space or comma-separated" +msgstr "" +"كلمة (كلمات) شائعة الاستخدام أو عبارة (عبارات) تستخدم لوصف الموضوع (فضاء أو " +"مفصولة بفواصل" + +msgid "Timezone" +msgstr "وحدة زمنية" + +msgid "No Group" +msgstr "لا توجد مجموعة" + +msgid "All contents" +msgstr "جميع المحتويات" + +msgid "No Organization Info" +msgstr "لا معلومات على المنظمة" + +msgid "Forgot Username" +msgstr "نسيت اسم المستخدم" + +msgid "" +"Enter your email address and click the submit button.
          Your username " +"will be sent to you." +msgstr "" +"أدخل عنوان بريدك الإلكتروني وانقر على زر إرسال.
          سيتم إرسال اسم " +"المستخدم إليك." + +msgid "Create Profile" +msgstr "إنشاء ملف شخصي" + +msgid "Profile of " +msgstr "الملف الشخصى " + +msgid "Not provided." +msgstr "غير متوفر." + +msgid "Position" +msgstr "موضع" + +msgid "Organization" +msgstr "تنظيم" + +msgid "Not provided" +msgstr "غير مزود" + +msgid "Fax" +msgstr "فاكس" + +msgid "User layers WMS GetCapabilities document" +msgstr "احصل على مستند القدرات لطبقات المستخدم لبرنامج إدارة المستودعات" + +msgid "Message User" +msgstr "رسالة المستخدم" + +msgid "Edit profile" +msgstr "تعديل الملف الشخصي" + +msgid "Connected social accounts" +msgstr "حسابات اجتماعية متصلة" + +msgid "Associated e-mails" +msgstr "رسائل البريد الإلكتروني المرتبطة" + +msgid "Set/Change password" +msgstr "تعيين / تغيير كلمة المرور" + +msgid "My Activities" +msgstr "نشاطاتي" + +msgid "Notifications" +msgstr "إشعارات" + +msgid "Announcements" +msgstr "الإعلانات" + +msgid "Invite Users" +msgstr "دعوة المستخدمين" + +msgid "User Activities" +msgstr "نشاطات المستخدم" + +msgid "Resources" +msgstr "موارد" + +msgid "Edit Your Profile" +msgstr "تحرير ملفك الشخصي" + +msgid "Edit Profile for" +msgstr "تعديل الملف الشخصي ل" + +msgid "Change your avatar" +msgstr "غير صورتك الشخصية" + +msgid "Update profile" +msgstr "تحديث الملف الشخصي" + +msgid "Explore People" +msgstr "تصفح الأشخاص" + +#, fuzzy, python-brace-format +#| msgid "Your username for " +msgid "Your username for {site.name}" +msgstr "اسم المستخدم الخاص بك ل " + +msgid "Your username has been emailed to you." +msgstr "تم إرسال اسم المستخدم الخاص بك إليك عبر البريد الإلكتروني." + +msgid "No user could be found with that email address." +msgstr "لا يمكن العثور على أي مستخدم باستخدام عنوان البريد الإلكتروني هذا." + +msgid "You are not permitted to save or edit this resource." +msgstr "غير مسموح لك بحفظ أو تعديل هذا المورد." + +#, fuzzy +#| msgid "You are not permitted to save or edit this resource." +msgid "You are not authorized to download this resource." +msgstr "غير مسموح لك بحفظ أو تعديل هذا المورد." + +msgid "" +"No files have been found for this resource. Please, contact a system " +"administrator." +msgstr "لم يتم العثور على أية ملفات لهذا المورد. الرجاء الاتصال بمسؤول النظام." + +msgid "No files found." +msgstr "وثائق وجدت." + +#, fuzzy +#| msgid "Not provided" +msgid "Not Authorized" +msgstr "غير مزود" + +#, fuzzy, python-brace-format +#| msgid "Some error occurred while trying to access the uploaded schema: %s" +msgid "Error occurred while executin the operation: '{_request.func_name}'" +msgstr "حدث بعض الأخطاء أثناء محاولة الوصول إلى المخطط الذي تم تحميله: %s" + +#, python-brace-format +msgid "Could not find the operation name: '{_request.func_name}'" +msgstr "" + +msgid "failed" +msgstr "" + +#, fuzzy +#| msgid "Warning" +msgid "running" +msgstr "تحذير" + +msgid "finished" +msgstr "" + +#, fuzzy +#| msgid "GeoNode Themes Library" +msgid "GeoNode Resource Processing Library" +msgstr "مواضيع مكتبة GeoNode" + +msgid "Disabling this Task will make the Processing Workflow to skip it." +msgstr "" + +msgid "Processing Workflow Tasks" +msgstr "" + +msgid "Session is Expired. Please login again!" +msgstr "انتهت الجلسة. الرجاد الدخول على الحساب من جديد!" + +msgid "Permissions successfully updated!" +msgstr "" + +#, python-brace-format +msgid "" +"User {username} has download permissions but cannot access the resource. " +"Please update permission consistently!" +msgstr "" + +#, fuzzy +#| msgid "Updating Permissions..." +msgid "Error updating permissions :(" +msgstr "جارٍ تحديث الصلاحيات ..." + +#, fuzzy +#| msgid "You do not have permissions for this layer." +msgid "You are not allowed to change permissions for this resource" +msgstr "لم يكن لديك صلاحيات لهذه الطبقة." + +msgid "Security Rules Cache Refreshed!" +msgstr "" + +#, fuzzy +#| msgid "You are not permitted to save or edit this resource." +msgid "You cannot modify this resource!" +msgstr "غير مسموح لك بحفظ أو تعديل هذا المورد." + +#, fuzzy +#| msgid "Service rescanned successfully" +msgid "Attributes/Stats Refreshed Successfully!" +msgstr "تم إعادة فحص الخدمة بنجاح" + +msgid "GeoWebCache Tiled Layer Emptied!" +msgstr "" + +#, fuzzy +#| msgid "Sync permissions immediately" +msgid "Wrong permissions specification" +msgstr "مزامنة الأذونات على الفور" + +#, fuzzy +#| msgid "Identification" +msgid "error delivering notification" +msgstr "هوية" + +msgid "Paired WMS/WFS/WCS" +msgstr "يقترن WMS/WFS/WCS" + +msgid "Web Map Service" +msgstr "خدمة خرائط الويب" + +msgid "ArcGIS REST MapServer" +msgstr "خادم خريطة ArcGIS REST" + +msgid "ArcGIS REST ImageServer" +msgstr "ArcGIS REST ImageServer" + +msgid "GeoNode (Web Map Service)" +msgstr "GeoNode (خدمة خريطة الويب)" + +msgid "Service URL" +msgstr "عنوان الخدمة" + +msgid "Service Type" +msgstr "نوع الخدمة" + +#, python-format +msgid "Service %(url)s is already registered" +msgstr "الخدمة%(url)s مسجلة بالفعل" + +#, python-format +msgid "Could not connect to the service at %(url)s" +msgstr "لا يمكن الاتصال بالخدمة على%(url)s" + +#, python-format +msgid "Found service of type %(found_type)s instead of %(service_type)s" +msgstr "تم العثور على خدمة من النوع%(found_type)s بدلاً من %(service_type)s" + +msgid "Local" +msgstr "محلي" + +msgid "Cascaded" +msgstr "تتالي" + +msgid "Harvested" +msgstr "حصاد" + +msgid "Indexed" +msgstr "مفهرس" + +msgid "Live" +msgstr "مباشر" + +msgid "OpenGeoPortal" +msgstr "OpenGeoPortal" + +msgid "Remote service unavailable" +msgstr "الخدمة عن بعد غير متوفرة" + +msgid "Service unavailable" +msgstr "الخدمة غير متوفرة" + +msgid "Could not contact" +msgstr "لا يمكن الاتصال" + +msgid "Contact" +msgstr "اتصال" + +msgid "Service Resources" +msgstr "موارد الخدمة" + +#, fuzzy +#| msgid "Resource is already being processed" +msgid "Resource harvesting in progress..." +msgstr "المورد قيد المعالجة بالفعل" + +msgid "No resources have been imported yet." +msgstr "لم يتم استيراد أي موارد حتى الآن." + +msgid "previous" +msgstr "السابق" + +msgid "Next" +msgstr "التالي" + +msgid "next" +msgstr "التالي" + +msgid "Connecting to service..." +msgstr "جارٍ الاتصال بالخدمة ..." + +msgid "Edit Service Metadata" +msgstr "تعديل البيانات الوصفية للخدمة" + +msgid "Import Service Resources" +msgstr "استيراد موارد الخدمة" + +msgid "Remove Service" +msgstr "إزالة الخدمة" + +msgid "Edit Service" +msgstr "تعديل الخدمة" + +msgid "Services" +msgstr "الخدمات" + +msgid "Add Remote Service" +msgstr "إضافة خدمة عن بعد" + +msgid "Remote Services" +msgstr "الخدمات عن بعد" + +msgid "No services registered" +msgstr "لا توجد خدمات مسجلة" + +msgid "Register Service" +msgstr "تسجيل خدمة" + +msgid "Register New Service" +msgstr "تسجيل خدمة جديدة" + +msgid "Remove Remote Service" +msgstr "إزالة الخدمة عن بعد" + +msgid "Are you sure you want to remove" +msgstr "هل أنت متأكد من أنك تريد إزالة" + +msgid "Deleting service and its associated resources..." +msgstr "جارٍ حذف الخدمة والموارد المرتبطة بها ..." + +msgid "Import resources" +msgstr "استيراد الموارد" + +msgid "no" +msgstr "لا" + +msgid "resources can be imported" +msgstr "يمكن استيراد الموارد" + +msgid "- These will be cascaded through your local geoserver instance" +msgstr "- سيتم تتابع هذه من خلال مثيلك الجغرافي المحلي" + +msgid "Clear Filter" +msgstr "مسح فلتر" + +msgid "Back to service details" +msgstr "العودة إلى تفاصيل الخدمة" + +msgid "Import Resources" +msgstr "استيراد الموارد" + +msgid "Previous" +msgstr "السابق" + +msgid "All resources have already been imported" +msgstr "تم بالفعل استيراد جميع الموارد" + +msgid "Harvesting resources..." +msgstr "حصاد الموارد ..." + +msgid "Re-scan Service for new Resources" +msgstr "إعادة فحص الخدمة للحصول على موارد جديدة" + +msgid "Service registered successfully" +msgstr "تم تسجيل الخدمة بنجاح" + +msgid "The selected resources are being imported" +msgstr "يتم استيراد الموارد المحددة" + +msgid "The selected resources have been imported" +msgstr "تم استيراد الموارد المحددة" + +#, fuzzy, python-brace-format +#| msgid "Resource {} is being processed" +msgid "Resource {resource_id} is being processed" +msgstr "المورد {} قيد المعالجة" + +msgid "Service rescanned successfully" +msgstr "تم إعادة فحص الخدمة بنجاح" + +#, fuzzy +#| msgid "You are not permitted to save or edit this resource." +msgid "You dont have enougth rigths to see the resource detail" +msgstr "غير مسموح لك بحفظ أو تعديل هذا المورد." + +msgid "You are not permitted to change this service." +msgstr "غير مسموح لك بتغيير هذه الخدمة." + +msgid "You are not permitted to remove this service." +msgstr "لا يسمح لك بإزالة هذه الخدمة." + +#, fuzzy, python-brace-format +#| msgid "Service {} has been deleted" +msgid "Service {service.name} has been deleted" +msgstr "تم حذف الخدمة {}" + +msgid "created" +msgstr "تم الإنشاء" + +msgid "deleted" +msgstr "تم الحذف" + +msgid "updated" +msgstr "تم التحديث" + +msgid "added a comment" +msgstr "إضافة تعليق" + +msgid "updated a comment" +msgstr "تحديث تعليق" + +msgid "uploaded" +msgstr "تم رفعها" + +msgid "Recent activity" +msgstr "النشاط الأخير" + +msgid "Layers" +msgstr "طبقات" + +msgid "to" +msgstr "إلى" + +msgid "on" +msgstr "في" + +msgid "Not Permitted" +msgstr "لا يسمح" + +msgid "" +"\n" +" You are not allowed to perform this operation.\n" +" " +msgstr "" +"\n" +"‫غير مسموح لك بتنفيذ هذه العملية." + +msgid "Please verify that you are logged in as the correct user." +msgstr "يرجى التحقق من أنك قمت بتسجيل الدخول باسم المستخدم الصحيح." + +msgid "Please login or register and retry." +msgstr "الرجاء الدخول أو التسجيل وإعادة المحاولة." + +msgid "Page Not Found" +msgstr "الصفحة غير موجودة" + +msgid "" +"\n" +" The page you requested does not exist. Perhaps you are using an " +"outdated bookmark?\n" +" " +msgstr "" +"\n" +"‫الصفحة التي طلبتها غير موجودة. ربما كنت تستخدم إشارة مرجعية قديمة?" + +msgid "Toggle navigation" +msgstr "تبديل الإبحار" + +msgid "People" +msgstr "الاشخاص" + +msgid "Register" +msgstr "تسجيل" + +msgid "Sign in" +msgstr "تسجيل الدخول" + +msgid "You are using an outdated browser that is not supported by GeoNode." +msgstr "أنت تستخدم متصفحا قديما غير مدعوم من قبل GeoNode." + +msgid "" +"Please use a modern browser like Mozilla Firefox, Google " +"Chrome or Safari." +msgstr "" +"الرجاء استخدام متصفح حديث مثل Mozilla Firefox أو Google " +"Chrome أو Safari." + +msgid "There was a problem loading this page" +msgstr "كانت هناك مشكلة تحميل لهذه الصفحة" + +msgid "" +"\n" +" Please contact your GeoNode administrator (they may have " +"received an email automatically if they configured it properly).\n" +" If you are the site administrator, enable debug mode to see the " +"actual error and fix it or file an issue in GeoNode's issue tracker\n" +" " +msgstr "" +"\n" +"\n" +"يرجى الاتصال بمسؤول GeoNode (قد يكون قد تلقى رسالة إلكترونية تلقائيا إذا قام " +"بتهيئته بشكل صحيح).\n" +"\n" +"\n" +"\n" +"إذا كنت مشرف الموقع، فيمكنك تمكين وضع التصحيح للاطلاع على الخطأ الفعلي " +"وإصلاحه أو تقديم طلب في تعقب المشكلة فيGeoNode " + +msgid "Add User" +msgstr "اضافة مستخدم" + +msgid "Powered by" +msgstr "مدعوم من" + +msgid "version" +msgstr "نسخة" + +msgid "Developers" +msgstr "المطورين" + +msgid "Username" +msgstr "اسم المستخدم" + +msgid "Password" +msgstr "كلمة المرور" + +msgid "Remember Me" +msgstr "تذكرني" + +msgid "Menu" +msgstr "القائمة" + +msgid "Recent Activity" +msgstr "النشاط الأخير" + +msgid "Inbox" +msgstr "البريد الوارد" + +msgid "Help" +msgstr "مساعدة" + +msgid "Log out" +msgstr "تسجيل الخروج" + +msgid "Info" +msgstr "معلومات" + +msgid "Granules" +msgstr "حبيبات" + +msgid "Share" +msgstr "شارك هذا" + +msgid "Ratings" +msgstr "تصنيفات" + +msgid "Exif" +msgstr "عن الصورة" + +msgid "Set permissions on selected resources" +msgstr "تعيين الصلاحيات على الموارد المختارة" + +msgid "Apply Changes" +msgstr "تطبيق التغييرات" + +msgid "total" +msgstr "مجموع" + +msgid "Add Comment" +msgstr "إضافة تعليق" + +msgid "Log in to add a comment" +msgstr "تسجيل الدخول لإضافة تعليق" + +msgid "By" +msgstr "بواسطة" + +msgid "Delete" +msgstr "حذف" + +msgid "Submit Comment" +msgstr "إرسال تعليق" + +msgid "Who can view it?" +msgstr "من يمكنه مشاهدته؟" + +msgid "Anyone" +msgstr "أي شخص" + +msgid "The following users:" +msgstr "المستخدمون التالين:" + +msgid "The following groups:" +msgstr "المجموعات التالية:" + +msgid "Who can download it?" +msgstr "من يمكنهم تحميله؟" + +msgid "Who can change metadata for it?" +msgstr "من يمكنه تغيير البيانات الوصفية لذلك؟" + +#, fuzzy +#| msgid "Who can edit data for this layer?" +msgid "Who can edit data for this dataset?" +msgstr "من يستطيع تعديل البيانات لهذه الطبقة؟" + +#, fuzzy +#| msgid "Who can edit styles for this layer?" +msgid "Who can edit styles for this dataset?" +msgstr "من يمكنه تعديل أنماط لهذه الطبقة؟" + +msgid "" +"Who can manage it? (update, delete, change permissions, publish/unpublish it)" +msgstr "من يمكنه إدارتها؟ (تحديث، حذف، أذونات التغيير، نشر / إلغاء نشره)" + +msgid "Set permissions for this resource" +msgstr "تعيين صلاحيات لهذا المورد" + +#, fuzzy +#| msgid "Explore Groups" +msgid "People and Groups" +msgstr "تصفح المجموعات" + +msgid "Geo Limits" +msgstr "الحدود الجغرافية" + +#, fuzzy +#| msgid "User" +#| msgid_plural "Users" +msgid "Users" +msgstr "المستخدم" + +#, fuzzy +msgid "Search" +msgstr "بحث" + +msgid "Show Geo Limits" +msgstr "إظهار الحدود الجغرافية" + +#, fuzzy +#| msgid "Upload Documents" +msgid "Upload Geo Limits" +msgstr "تحميل المستندات" + +msgid "Save Geo Limits" +msgstr "حفظ الحدود الجغرافية" + +#, fuzzy +#| msgid "Delete from Favorites" +msgid "Delete Geo Limits" +msgstr "حذف من المفضلات" + +msgid "Message box" +msgstr "صندوق الرسائل" + +msgid "OK" +msgstr "حسنا" + +msgid "Mandatory files : SHP , DBF" +msgstr "الملفات الإلزامية : SHP ، DBF" + +msgid "" +"Upload a ZIP file containing an ESRI Shapefile. If the ZIP provides also a ." +"prj file, you don't have to specify the EPSG SRID" +msgstr "" +"تحميل ملف ZIP يحتوي على ملف شكل ESRI. إذا كان ZIP يوفر أيضا ملف .prj، لم يكن " +"لديك لتحديد SRID EPSG" + +#, fuzzy +#| msgid "Choose Files" +msgid "Choose" +msgstr "اختيار المستخدمين" + +msgid "Choose a file..." +msgstr "اختيار المستخدمين." + +msgid "Reset" +msgstr "إعادة تعيين" + +msgid "Only the SRID number without the 'EPSG:' prefix!" +msgstr "فقط رقم SRID بدون بادئة \"EPSG:\" !" + +msgid "Default : 4326" +msgstr "الافتراضي : 4326" + +msgid "Set your desired encoding UTF-8, Big5, Big5-HKSCS ..." +msgstr "تعيين الترميز المطلوب UTF-8، Big5، Big5-HKSCS ..." + +msgid "Load SHP-ZIP" +msgstr "تحميل SHP-ZIP" + +msgid "" +"

          This will remove the Geo Limits currently drawn on the map.

          In " +"order to store the Geo Limits you will need to save them " +"anyway.

          Do you want to proceed?

          " +msgstr "" +"

          سيؤدي ذلك إلى إزالة الحدود الجغرافية المرسومة حاليًا على الخريطة.

          من " +"أجل تخزين حدود الجغرافية سوف تحتاج إلى حفظها على أي حال.

          هل تريد المتابعة؟

          " + +#, fuzzy +#| msgid "" +#| "

          This will override the current stored Geo Limits on the DB.

          To " +#| "apply them you will need to click on \\" +msgid "" +"

          This will override the current stored Geo Limits on the DB.

          To " +"apply them you will need to click on Apply Changes button " +"anyway.

          WARNING: This operation cannot be reverted!

          Do you want to proceed?

          " +msgstr "" +"

          سيتجاوز هذا الحدود الجغرافية المخزنة الحالية على DB.

          لتطبيقها سوف " +"تحتاج إلى النقر على \\" + +msgid "Save Geo Limit" +msgstr "حفظ الحد الجغرافي" + +#, fuzzy +#| msgid "" +#| "

          Geometry successfully saved!

          To apply them you " +#| "will need to click on \\" +msgid "" +"

          Geometry successfully saved!

          To apply them you " +"will need to click on Apply Changes button.

          " +msgstr "" +"

          الهندسة المحفوظة بنجاح!

          لتطبيقها سوف تحتاج إلى " +"النقر على \\" + +msgid "

          Error while trying to save the Geometry!

          " +msgstr "

          خطأ أثناء محاولة حفظ الهندسة!

          " + +msgid "Load Geo Limits" +msgstr "تحميل الحدود الجغرافية" + +msgid "Load Geo Limit" +msgstr "تحميل الحد الجغرافي" + +msgid "

          No Geometry found!

          " +msgstr "

          لم يتم العثور على هندسة!

          " + +#, fuzzy +#| msgid "" +#| "

          This will permanently remove the Geo Limits from the " +#| "DB.

          To apply them you will need to click on \\" +msgid "" +"

          This will permanently remove the Geo Limits from the DB." +"

          To apply them you will need to click on Apply Changes button.

          Do you want to proceed?

          " +msgstr "" +"

          سيؤدي ذلك إلى إزالة الحدود الجغرافية من DB بشكل دائم.

          لتطبيقها سوف تحتاج إلى النقر على \\" + +msgid "Delete Geo Limit" +msgstr "حذف الحد الجغرافي" + +msgid "

          Geo Limitsy successfully deleted!

          " +msgstr "

          Geo Limitsy حذف بنجاح!

          " + +msgid "" +"

          Error occurred while trying to delete Geo Limits:

          " +msgstr "

          حدث خطأ أثناء محاولة حذف الحدود الجغرافية:

          " + +msgid "Choose users..." +msgstr "اختيار المستخدمين ..." + +msgid "Choose groups..." +msgstr "اختيار مجموعات ..." + +msgid "Placeholder for status-message" +msgstr "عنصر نائب لحالة الرسالة" + +msgid "Placeholder for status-message-body" +msgstr "عنصر نائب لحالة الرسائل النصية" + +msgid "About GeoNode" +msgstr "حول GeoNode" + +msgid "" +"GeoNode is a geospatial content management system, a platform for the " +"management and publication of geospatial data. It brings together mature and " +"stable open-source software projects under a consistent and easy-to-use " +"interface allowing non-specialized users to share data and create " +"interactive maps." +msgstr "" +"GeoNode هو نظام لإدارة المحتوى الجغرافي المكاني ، ومنصة لإدارة ونشر البيانات " +"الجغرافية المكانية. فهو يجمع بين مشاريع البرامج مفتوحة المصدر ناضجة وناضجة " +"تحت واجهة متسقة وسهلة الاستخدام مما يسمح للمستخدمين غير المتخصصين بمشاركة " +"البيانات وإنشاء خرائط تفاعلية." + +msgid "" +"Data management tools built into GeoNode allow for integrated creation of " +"data, metadata, and map visualizations. Each dataset in the system can be " +"shared publicly or restricted to allow access to only specific users. Social " +"features like user profiles and commenting and rating systems allow for the " +"development of communities around each platform to facilitate the use, " +"management, and quality control of the data the GeoNode instance contains." +msgstr "" +"تتيح أدوات إدارة البيانات المدمجة في GeoNode إنشاء تصورات متكاملة للبيانات " +"والبيانات الوصفية وتصور الخريطة. يمكن مشاركة كل مجموعة بيانات في النظام بشكل " +"عام أو تقييدها للسماح بالوصول إلى مستخدمين محددين فقط . تسمح الميزات " +"الاجتماعية مثل ملفات تعريف المستخدمين وأنظمة التعليق والتصنيف بتطوير " +"المجتمعات حول كل نظام أساسي لتسهيل استخدام وإدارة ومراقبة جودة البيانات التي " +"يحتوي عليها مثيل GeoNode." + +msgid "" +"It is also designed to be a flexible platform that software developers can " +"extend, modify or integrate against to meet requirements in their own " +"applications." +msgstr "" +"كما أنه مصمم ليكون نظامًا مرنًا حيث يستطيع مطورو البرامج تمديده أو تعديله أو " +"دمجه لتلبية المتطلبات في تطبيقاتهم الخاصة." + +msgid "Account Inactive" +msgstr "حساب غير نشط" + +msgid "This account is inactive." +msgstr "هذا الحساب غير نشط." + +msgid "Account Pending Approval" +msgstr "الحساب قيد الموافقة" + +#, python-format +msgid "" +"We have sent the administrators a notice to approve your account associated " +"with %(email)s. If the account is approved, you will receive a " +"confirmation notice." +msgstr "" +"لقد أرسلنا للمشرفين إشعارا بالموافقة على حسابك المرتبط ب %(email)sإذا " +"تمت الموافقة على الحساب، سوف تتلقى إشعار تأكيد." + +msgid "Account" +msgstr "الحساب" + +msgid "E-mail Addresses" +msgstr "عناوين البريد الإلكتروني" + +msgid "The following e-mail addresses are associated with your account:" +msgstr "عناوين البريد الإلكتروني التالية مرتبطة بحسابك:" + +msgid "Verified" +msgstr "تحقق منه" + +msgid "Unverified" +msgstr "غير متحقق منه" + +msgid "Primary" +msgstr "الأولية" + +msgid "Make Primary" +msgstr "إتخذ الاولوية" + +msgid "Re-send Verification" +msgstr "إعادة إرسال التحقق" + +msgid "Warning:" +msgstr "تحذير:" + +msgid "" +"You currently do not have any e-mail address set up. You should really add " +"an e-mail address so you can receive notifications, reset your password, etc." +msgstr "" +"ليس لديك حاليًا أي عنوان بريد إلكتروني تم إعداده. يجب عليك بالفعل إضافة عنوان " +"بريد إلكتروني حتى تتمكن من تلقي الإشعارات وإعادة تعيين كلمة المرور الخاصة " +"بك ، إلخ." + +msgid "Add E-mail Address" +msgstr "إضافة عنوان البريد الإلكتروني" + +msgid "Add E-mail" +msgstr "إضافة البريد الإلكتروني" + +msgid "Do you really want to remove the selected e-mail address?" +msgstr "هل تريد حقًا إزالة عنوان البريد الإلكتروني المحدد؟" + +#, fuzzy, python-format +#| msgid "" +#| "Hello from %(site_name)s!\n" +#| "\n" +#| "You're receiving this e-mail because user %(user_display)s has given " +#| "yours as an e-mail address to connect their account.\n" +#| "\n" +#| "To confirm this is correct, go to %(activate_url)s\n" +msgid "" +"Hello from %(site_name|urlencode)s!\n" +"\n" +"You're receiving this e-mail because user %(user_display)s has given yours " +"as an e-mail address to connect their account.\n" +"\n" +"To confirm this is correct, go to %(activate_url)s\n" +msgstr "" +"مرحبًا من%(site_name)s!\n" +"\n" +"أنت تتلقى هذه الرسالة الإلكترونية لأن المستخدم%(user_display)s قد أعطى عنوان " +"بريدك الإلكتروني للاتصال بحسابه.\n" +"\n" +"لتأكيد ان هذا صحيح ، انتقل إلى%(activate_url)s\n" +"\n" + +#, fuzzy, python-format +#| msgid "" +#| "Thank you from %(site_name)s!\n" +#| "%(site_domain)s" +msgid "" +"Thank you from %(site_name|urlencode)s!\n" +"%(site_domain)s" +msgstr "" +"شكرًا لك من%(site_name)s!\n" +"%(site_domain)s" + +#, fuzzy, python-format +#| msgid "Please confirm email address for %(site_name)s" +msgid "Please confirm email address for %(site_name|urlencode)s" +msgstr "الرجاء تأكيد عنوان البريد الإلكتروني ل %(site_name)s" + +#, fuzzy, python-format +#| msgid "You have been invited to sign up at %(site_name)s." +msgid "You have been invited to sign up at %(site_name|urlencode)s." +msgstr "لقد تمت دعوتك للتسجيل في%(site_name)s." + +#, fuzzy, python-format +#| msgid "Create an account on %(site_name)s" +msgid "Create an account on %(site_name|urlencode)s" +msgstr "إنشاء حساب علي%(site_name)s " + +msgid "" +"This is the email notification to confirm your password has been changed on" +msgstr "هذا هو الاعلام بالبريد الكتروني لتاكيد تغيير كلمه المرور الخاصة بك علي" + +msgid "Change password email notification" +msgstr "إشعار البريد الإلكتروني لتغيير كلمة المرور" + +#, fuzzy, python-format +#| msgid "" +#| "Hello from %(site_name)s!\n" +#| "\n" +#| "You're receiving this e-mail because you or someone else has requested a " +#| "password for your user account.\n" +#| "It can be safely ignored if you did not request a password reset. Click " +#| "the link below to reset your password." +msgid "" +"Hello from %(site_name|urlencode)s!\n" +"\n" +"You're receiving this e-mail because you or someone else has requested a " +"password for your user account.\n" +"It can be safely ignored if you did not request a password reset. Click the " +"link below to reset your password." +msgstr "" +"مرحبا من %(site_name)s!\n" +"\n" +"أنت تتلقى هذا البريد الإلكتروني لأنك أنت أو شخص آخر قد طلب كلمة مرور لحساب " +"المستخدم الخاص بك.\n" +" فإنه يمكن تجاهلها بأمان إذا لم تطلب إعادة تعيين كلمة المرور. اضغط على " +"الرابط أدناه لإعادة تعيين كلمة المرور الخاصة بك." + +#, python-format +msgid "In case you forgot, your username is %(username)s." +msgstr "في حال نسيت ، اسم المستخدم الخاص بك هو %(username)s." + +#, fuzzy, python-format +#| msgid "" +#| "Thank you for using %(site_name)s!\n" +#| "%(site_domain)s" +msgid "" +"Thank you for using %(site_name|urlencode)s!\n" +"%(site_domain)s" +msgstr "" +"شكرًا لك على استخدام%(site_name)s!\n" +"%(site_domain)s" + +#, fuzzy +#| msgid "Password reset" +msgid " Password reset" +msgstr "إعادة تعيين كلمة المرور" + +msgid "Confirm E-mail Address" +msgstr "أكد عنوانك الإلكتروني" + +#, python-format +msgid "" +"Please confirm that %(email)s is an e-mail " +"address for user %(user_display)s." +msgstr "" +"يرجى تأكيد أن%(email)s هو عنوان بريد " +"إلكتروني للمستخدم%(user_display)s." + +msgid "Confirm" +msgstr "أكد" + +#, python-format +msgid "" +"This e-mail confirmation link expired or is invalid. Please issue a new e-mail confirmation request." +msgstr "" +"انتهت صلاحية رابط تأكيد البريد الإلكتروني أو غير صالح. الرجاء إصدار طلب تأكيد بريد إلكتروني جديد ." + +msgid "Log in" +msgstr "تسجيل الدخول" + +msgid "Log in to an existing account" +msgstr "تسجيل الدخول إلى حساب موجود" + +#, python-format +msgid "" +"Please sign in with one\n" +" of your existing third party accounts. Or, sign up\n" +" for a %(site_name)s account and sign in below:" +msgstr "" +"يرجى تسجيل الدخول بواحد\n" +"       من حسابات الطرف الثالث الموجودة لديك. أو " +"اشترك \n" +"       لحساب%(site_name)s وسجل الدخول أدناه:" + +msgid "or" +msgstr "أو" + +#, python-format +msgid "" +"If you have not created an account yet, then please\n" +" sign up first." +msgstr "" +"إذا لم تقم بإنشاء حساب بعد ، فالرجاء\n" +" التسجيل أولاً." + +msgid "Forgot Password?" +msgstr "هل نسيت كلمة المرور؟" + +msgid "Sign In" +msgstr "تسجيل الدخول" + +msgid "Are you sure you want to log out?" +msgstr "هل أنت متأكد أنك تريد الخروج؟" + +msgid "Password change" +msgstr "تغيير كلمة السر" + +msgid "Password Change" +msgstr "تغيير كلمة السر" + +msgid "Change your password here" +msgstr "غير كلمة المرور هنا" + +msgid "Change my password" +msgstr "تغيير كلمة المرور الخاصة بي" + +msgid "Password reset" +msgstr "إعادة تعيين كلمة المرور" + +msgid "Password Reset" +msgstr "إعادة تعيين كلمة المرور" + +msgid "Forgotten your password?" +msgstr "هل نسيت كلمة المرور؟" + +msgid "" +"Enter your email address below, and we'll send you an email allowing you to " +"reset it." +msgstr "" +"أدخل عنوان بريدك الإلكتروني أدناه، وسنرسل لك رسالة إلكترونية تتيح لك إعادة " +"تعيينها." + +msgid "Reset my password" +msgstr "إعادة تعيين كلمة المرور الخاصة بي" + +#, python-format +msgid "" +"If you have any trouble resetting your password, contact us at %(THEME_ACCOUNT_CONTACT_EMAIL)s." +msgstr "" +"إذا كان لديك أي مشكلة إعادة تعيين كلمة المرور الخاصة بك، اتصل بنا على " +"%(THEME_ACCOUNT_CONTACT_EMAIL)s." + +msgid "" +"We have sent you an e-mail. Please contact us if you do not receive it " +"within a few minutes." +msgstr "" +"لقد أرسلنا لك رسالة بريد إلكتروني. يرجى الاتصال بنا إذا لم تتلقاها في غضون " +"دقائق قليلة." + +msgid "Change Password" +msgstr "غير كلمة السر" + +msgid "Bad Token" +msgstr "رمز سيء" + +#, python-format +msgid "" +"The password reset link was invalid, possibly because it has already been " +"used. Please request a new password reset." +msgstr "" +"رابط إعادة تعيين كلمة المرور غير صالح ، ربما لأنه تم استخدامه بالفعل. الرجاء " +"ارسل طلب إعادة تعيين كلمة مرور جديدة." + +msgid "change password" +msgstr "غير كلمة السر" + +msgid "Your password is now changed." +msgstr "تم تغيير كلمة المرور الخاصة بك الآن." + +msgid "Set Password" +msgstr "ضبط كلمة السر" + +msgid "Password Set" +msgstr "تعيين كلمة المرور" + +msgid "Set your password here" +msgstr "اضبط كلمة المرور هنا" + +msgid "Set password" +msgstr "ضع كلمة سر" + +msgid "Sign up" +msgstr "سجل" + +msgid "" +"Sign up with one\n" +" of your existing third party accounts" +msgstr "" +"الاشتراك بواحد\n" +"               من حسابات الطرف الثالث الموجودة لديك" + +msgid "Create a new local account" +msgstr "إنشاء حساب محلي جديد" + +msgid "Sign Up Closed" +msgstr "التسجيل مغلق" + +msgid "We are sorry, but the sign up is currently closed." +msgstr "نأسف ، لكن التسجيل مغلق حاليًا." + +msgid "Note" +msgstr "ملحوظة" + +#, python-format +msgid "you are already logged in as %(user_display)s." +msgstr "تم تسجيل دخولك بالفعل كـ%(user_display)s." + +msgid "Verify your email address" +msgstr "تحقق من عنوان بريدك الإلكتروني" + +msgid "" +"We have sent an email to you for verification. Follow the\n" +" link provided to finalize the signup process. Please contact us if you " +"do\n" +" not receive it within a few minutes" +msgstr "" +"لقد أرسلنا إليك رسالة إلكترونية إلى ٪ (email) s للتحقق منها. اتبع " +"الرابط المقدم لوضع اللمسات الأخيرة على عملية الاشتراك" + +msgid "Verify Your E-mail Address" +msgstr "تحقق من عنوان البريد الإلكتروني الخاص بك" + +msgid "" +"This part of the site requires us to verify that\n" +"you are who you claim to be. For this purpose, we require that you\n" +"verify ownership of your e-mail address. " +msgstr "" +"هذا الجزء من الموقع يتطلب منا التحقق من أنك\n" +"أنت الذي تدعي أنك هو. لهذا الغرض ، نطلب منك\n" +"تحقق من ملكية عنوان البريد الإلكتروني الخاص بك. " + +msgid "" +"We have sent an e-mail to you for\n" +"verification. Please click on the link inside this e-mail. Please\n" +"contact us if you do not receive it within a few minutes." +msgstr "" +"لقد أرسلنا لك بريدًا إلكترونيًا من أجل\n" +"التحقق. الرجاء الضغط على الرابط داخل هذا البريد الإلكتروني. رجاء\n" +"اتصل بنا إذا لم تستلمه خلال بضع دقائق." + +#, python-format +msgid "" +"Note: you can still change your e-" +"mail address." +msgstr "" +" ملاحظة: لا يزال بإمكانك تغيير " +"عنوان البريد الإلكتروني ." + +msgid "My Activity feed" +msgstr "خلاصة أنشطتي" + +msgid "Activity Feed" +msgstr "خلاصة الأنشطة" + +msgid "Are you sure you want to run the selected" +msgstr "هل أنت متأكد أنك تريد تشغيل المحدد" + +msgid "Yes, I'm sure" +msgstr "نعم، أنا متأكد" + +msgid "GeoNode site admin" +msgstr "مشرف موقع GeoNode" + +msgid "GeoNode administration" +msgstr "إدارة GeoNode" + +msgid "Delete Announcement?" +msgstr "هل تريد حذف الإعلان؟" + +msgid "Are you sure you want to delete this announcement?" +msgstr "هل انت متاكد انك تريد حذف هذا الاعلان؟" + +msgid "Edit Announcement" +msgstr "تعديل إعلان" + +msgid "Create Announcement" +msgstr "إنشاء إعلان" + +msgid "New Announcement" +msgstr "إعلان جديد" + +msgid "Level" +msgstr "مستوى" + +msgid "Announcement" +msgstr "إعلان" + +msgid "Published From" +msgstr "نشرت من" + +#, python-format +msgid "" +"\n" +" Published from %(publish_start)s to %(publish_end)s.\n" +" " +msgstr "" +"\n" +"\n" +"تم النشر من %(publish_start)s إلى %(publish_end)s" + +#, python-format +msgid "Announcements : %(announcement)s" +msgstr "إعلانات: %(announcement)s" + +msgid "Back to edit your profile information" +msgstr "الرجوع الى تعديل معلومات ملفك الشخصي" + +msgid "Your current avatar: " +msgstr "الصورة الشخصية الحالية " + +msgid "You haven't uploaded an avatar yet. Please upload one now." +msgstr "لم يتم تحميل صورة شخصية بعد. الرجاء تحميل واحدة الآن." + +msgid "Upload New Image" +msgstr "تحميل صورة جديدة" + +msgid "Choose new Default" +msgstr "إختر إفتراضي جديد" + +msgid "Delete Your Avatar" +msgstr "حذف الصورة الشخصية" + +msgid "Please select the avatars that you would like to delete." +msgstr "الرجاء تحديد الصور الشخصية التي تريد حذفها." + +#, python-format +msgid "" +"You have no avatars to delete. Please upload one now." +msgstr "" +"ليس لديك أي صور شخصية للحذف. الرجاء <\"a href=\"%(avatar_change_url)s> تحميل " +"ملف واحد الآن." + +msgid "Delete These" +msgstr "حذف هذه" + +msgid "GeoNode Search" +msgstr "بحث GeoNode" + +#, fuzzy +#| msgid "GeoExplorer" +msgid "Explore" +msgstr "GeoExplorer" + +msgid "Uploading..." +msgstr "تحميل ..." + +msgid "Upload in progress..." +msgstr "التحميل قيد التقدم ..." + +msgid "Processing..." +msgstr "معالجة..." + +msgid "Updating Thumbnail..." +msgstr "جارٍ تحديث الصورة المصغرة ..." + +msgid "Message. Do you want to proceed?" +msgstr "هل أنت متأكد من أنك تريد إزالة?" + +msgid "Some text in the modal" +msgstr "بعض النصوص في الوسائط" + +#, fuzzy +#| msgid "Harvesting resources..." +msgid "Processing Resource..." +msgstr "حصاد الموارد ..." + +msgid "Information for Developers" +msgstr "معلومات للمطورين" + +msgid "Useful information for developers interested in GeoNode." +msgstr "معلومات مفيدة للمطورين المهتمين ب GeoNode." + +#, fuzzy, python-format +#| msgid "" +#| "\n" +#| "

          GeoNode is an open service " +#| "built on open source software. We encourage you to build new applications " +#| "using the components and resources it provides. This page is a starting " +#| "point for developers interesting in taking full advantage of GeoNode. It " +#| "also includes links to the project's source code so anyone can build and " +#| "customize their own GeoNode.

          \n" +#| "\n" +#| "

          GeoNode Software

          \n" +#| "\n" +#| "

          All the code that runs GeoNode is open source. The code is " +#| "available at http://github." +#| "com/GeoNode/geonode/. The issue tracker for the project is at http://github.com/GeoNode/" +#| "geonode/issues.

          \n" +#| "\n" +#| "

          GeoNode is built using several open source projects, each with its " +#| "own community. If you are interested in contributing new features to the " +#| "GeoNode, we encourage you to do so by contributing to one of the projects " +#| "on which it is built:

          \n" +#| "
            \n" +#| "
          • GeoServer - Standards " +#| "based server for geospatial information
            • \n" +#| "
          • GeoWebCache - Cache " +#| "engine for WMS Tiles
          • OpenLayers - Pure JavaScript library powering the maps of GeoExt\n" +#| "
          • pycsw - CSW, OpenSearch and " +#| "OAI-PMH metadata catalogue server
            • \n" +#| "
          \n" +#| "\n" +#| "

          What are OGC Services?

          \n" +#| "

          The data in this application is served using open standards " +#| "endorsed by ISO and the Open " +#| "Geospatial Consortium; in particular, WMS (Web Map Service) is used " +#| "for accessing maps, WFS (Web Feature Service) is used for accessing " +#| "vector data, and WCS (Web Coverage Service) is used for accessing raster " +#| "data. WMC (Web Map Context Documents) is used for sharing maps. You can " +#| "use these services in your own applications using libraries such as " +#| "OpenLayers, GeoTools, and OGR (all of which are open-source software and " +#| "available at zero cost). Additionally, CSW (Catalog Service for the Web) " +#| "supports access to collections of descriptive information (metadata) " +#| "about data and services.

          \n" +#| "\n" +#| "

          What is GeoWebCache?

          \n" +#| "

          GeoWebCache provides mapping tiles that are compatible with a " +#| "number of mapping engines, including Google Maps, Bing Maps and " +#| "OpenLayers. All the data hosted by GeoNode is also available through " +#| "GeoWebCache. GeoWebCache improves on WMS by caching data and providing " +#| "more responsive maps.

          \n" +#| "\n" +#| "

          CSW Example Code

          \n" +#| "

          To interact with GeoNode's CSW you can use any CSW client (QGIS " +#| "MetaSearch, GRASS, etc.). The following example illustrates a simple " +#| "invocation using the OWSLib Python package:

          \n" +#| "

          from owslib.csw import CatalogueServiceWeb

          \n" +#| "

          from owslib.fes import PropertyIsLike

          \n" +#| "

          csw = CatalogueServiceWeb('%(CATALOGUE_BASE_URL)s')\n" +#| "

          anytext = PropertyIsLike('csw:AnyText', 'birds')')\n" +#| "

          csw.getrecords2(constraints=[anytext])

          \n" +#| "

          print csw.results

          \n" +#| "

          print csw.records

          \n" +#| "\n" +#| "

          OpenLayers Example Code

          \n" +#| "\n" +#| "

          To include a GeoNode map layer in an OpenLayers map, first find " +#| "the name for that layer. This is found in the layer's name " +#| "field (not title) of the layer list. For this example, we " +#| "will use the Nicaraguan political boundaries background layer, whose name " +#| "is risk:nicaragua_admin. Then, create an instance of " +#| "OpenLayers.Layer.WMS:

          \n" +#| "

          var geonodeLayer = new OpenLayers.Layer.WMS(\"GeoNode Risk " +#| "Data\", \"http://demo.geonode.org/geoserver/wms\",{ layers: \"risk:" +#| "nicaragua_admin\" });

          \n" +#| "\n" +#| "

          Google Maps Example Code

          \n" +#| "

          To include a GeoNode map layer in a Google Map, include the layer " +#| "namein the URL template.

          \n" +#| "

          var tilelayer = new GTileLayer(null, null, null, " +#| "{tileUrlTemplate: 'http://demo.geonode.org/geoserver/gwc/service/gmaps?" +#| "layers=risk:nicaragua_admin&zoom={Z}&x={X}&y={Y}', isPng:" +#| "true, opacity:0.5 } );

          \n" +#| "\n" +#| "

          Shapefile/GeoJSON/GML Output

          \n" +#| "

          To get data from the GeoNode web services use the WFS protocol. " +#| "For example, to get the full Nicaraguan admin boundaries use:

          \n" +#| "

          http://demo.geonode.org/geoserver/wfs?request=GetFeature&" +#| "typeName=risk:nicaragua_admin&outputformat=SHAPE-ZIP

          \n" +#| "

          Changing output format to json, GML2, " +#| "GML3, or csv will get data in those formats. " +#| "The WFS protocol also can handle more precise queries, specifying a " +#| "bounding box or various spatial and non-spatial filters based on the " +#| "attributes of the data.

          \n" +#| "\n" +#| "

          GeoTools Example Code

          \n" +#| "

          Create a DataStore and extract a FeatureType from it, then run a " +#| "Query. It is all documented on the wiki at http://geotools.org/.

          \n" +#| " " +msgid "" +"\n" +"

          GeoNode is an open service built " +"on open source software. We encourage you to build new applications using " +"the components and resources it provides. This page is a starting point for " +"developers interesting in taking full advantage of GeoNode. It also includes " +"links to the project's source code so anyone can build and customize their " +"own GeoNode.

          \n" +"\n" +"

          GeoNode Software

          \n" +"\n" +"

          All the code that runs GeoNode is open source. The code is available " +"at http://github.com/GeoNode/" +"geonode/. The issue tracker for the project is at http://github.com/GeoNode/geonode/" +"issues.

          \n" +"\n" +"

          GeoNode is built using several open source projects, each with its " +"own community. If you are interested in contributing new features to the " +"GeoNode, we encourage you to do so by contributing to one of the projects on " +"which it is built:

          \n" +"
            \n" +"
          • GeoServer - Standards based " +"server for geospatial information
            • \n" +"
          • GeoWebCache - Cache engine " +"for WMS Tiles
          • OpenLayers - " +"Pure JavaScript library powering the maps of GeoExt
            • \n" +"
          • pycsw - CSW, OpenSearch and OAI-" +"PMH metadata catalogue server
            • \n" +"
          \n" +"\n" +"

          What are OGC Services?

          \n" +"

          The data in this application is served using open standards endorsed " +"by ISO and the Open Geospatial " +"Consortium; in particular, WMS (Web Map Service) is used for accessing " +"maps, WFS (Web Feature Service) is used for accessing vector data, and WCS " +"(Web Coverage Service) is used for accessing raster data. WMC (Web Map " +"Context Documents) is used for sharing maps. You can use these services in " +"your own applications using libraries such as OpenLayers, GeoTools, and OGR " +"(all of which are open-source software and available at zero cost). " +"Additionally, CSW (Catalog Service for the Web) supports access to " +"collections of descriptive information (metadata) about data and services.\n" +"\n" +"

          What is GeoWebCache?

          \n" +"

          GeoWebCache provides mapping tiles that are compatible with a number " +"of mapping engines, including Google Maps, Bing Maps and OpenLayers. All the " +"data hosted by GeoNode is also available through GeoWebCache. GeoWebCache " +"improves on WMS by caching data and providing more responsive maps.

          \n" +"\n" +"

          CSW Example Code

          \n" +"

          To interact with GeoNode's CSW you can use any CSW client (QGIS " +"MetaSearch, GRASS, etc.). The following example illustrates a simple " +"invocation using the OWSLib Python package:

          \n" +"

          from owslib.csw import CatalogueServiceWeb

          \n" +"

          from owslib.fes import PropertyIsLike

          \n" +"

          csw = CatalogueServiceWeb('%(CATALOGUE_BASE_URL)s')

          \n" +"

          anytext = PropertyIsLike('csw:AnyText', 'birds')')

          \n" +"

          csw.getrecords2(constraints=[anytext])

          \n" +"

          print csw.results

          \n" +"

          print csw.records

          \n" +"\n" +"

          OpenLayers Example Code

          \n" +"\n" +"

          To include a GeoNode map layer in an OpenLayers map, first find the " +"name for that layer. This is found in the layer's name field " +"(not title) of the layer list. For this example, we will use " +"the Nicaraguan political boundaries background layer, whose name is " +"risk:nicaragua_admin. Then, create an instance of OpenLayers." +"Layer.WMS:

          \n" +"

          var geonodeLayer = new OpenLayers.Layer.WMS(\"GeoNode Risk Data" +"\", \"http://demo.geonode.org/geoserver/wms\",{ layers: \"risk:" +"nicaragua_admin\" });

          \n" +"\n" +"

          Google Maps Example Code

          \n" +"

          To include a GeoNode map layer in a Google Map, include the dataset " +"name in the URL template.

          \n" +"

          var tilelayer = new GTileLayer(null, null, null, " +"{tileUrlTemplate: 'http://demo.geonode.org/geoserver/gwc/service/gmaps?" +"layers=risk:nicaragua_admin&zoom={Z}&x={X}&y={Y}', isPng:true, " +"opacity:0.5 } );

          \n" +"\n" +"

          Shapefile/GeoJSON/GML Output

          \n" +"

          To get data from the GeoNode web services use the WFS protocol. For " +"example, to get the full Nicaraguan admin boundaries use:

          \n" +"

          http://demo.geonode.org/geoserver/wfs?request=GetFeature&" +"typeName=risk:nicaragua_admin&outputformat=SHAPE-ZIP

          \n" +"

          Changing output format to json, GML2, " +"GML3, or csv will get data in those formats. The " +"WFS protocol also can handle more precise queries, specifying a bounding box " +"or various spatial and non-spatial filters based on the attributes of the " +"data.

          \n" +"\n" +"

          GeoTools Example Code

          \n" +"

          Create a DataStore and extract a FeatureType from it, then run a " +"Query. It is all documented on the wiki at http://geotools.org/." +"

          \n" +" " +msgstr "" +"‫\n" +"‬\n" +"‫‏‫

          GeoNode هي خدمة مفتوحة مبنية على " +"برنامج مفتوح المصدر. نحن نشجعك على إنشاء تطبيقات جديدة باستخدام المكونات " +"والموارد التي توفرها. هذه الصفحة هي نقطة انطلاق للمطورين المهتمين بالاستفادة " +"الكاملة من GeoNode. ويشمل أيضًا روابط إلى الكود المصدري للمشروع حتى يتمكن أي " +"شخص من بناء وتخصيص GeoNode الخاص به. \n" +"‬\n" +"‏‫‬\n" +"‬\n" +"‫‏‫\n" +"‬\n" +"‏‫‬\n" +"‬\n" +"‫‏‫    

          GeoNode تطبيق

          \n" +"‬\n" +"‏‫‬\n" +"‬\n" +"‫‏‫\n" +"‬\n" +"‏‫‬\n" +"‬\n" +"‫‏‫    

          كل الشفرة التي تدير GeoNode مفتوحة المصدر. الكود متاح على http://github.com/GeoNode/geonode/ . يتوفر متتبع المشكلات للمشروع على http://github.com/GeoNode/geonode/issues .

          \n" +"‬\n" +"‏‫‬\n" +"‬\n" +"‫‏‫\n" +"‬\n" +"‏‫‬\n" +"‬\n" +"‫‏‫    

          تم تصميم GeoNode باستخدام عدة مشاريع مفتوحة المصدر ، لكل منها " +"مجتمعها الخاص. إذا كنت مهتمًا بالمساهمة بميزات جديدة في GeoNode ، فنحن نشجعك " +"على القيام بذلك من خلال المساهمة في أحد المشاريع التي تم تصميمها عليها: \n" +"‬\n" +"‏‫‬\n" +"‬\n" +"‫‏‫