Skip to content

change nginx config to show real ip and limit crawlers spam #159

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aydenjahola wants to merge 1 commit into
base: master
Choose a base branch
from
Open

change nginx config to show real ip and limit crawlers spam #159

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 145 additions & 33 deletions jobs/services/wiki/mediawiki.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,27 @@ job "mediawiki" {
"traefik.http.routers.rbwiki.rule=Host(`${NOMAD_META_domain}`) || Host(`wiki.rb.dcu.ie`)",
"traefik.http.routers.rbwiki.entrypoints=web,websecure",
"traefik.http.routers.rbwiki.tls.certresolver=rb",
"traefik.http.routers.rbwiki.middlewares=rbwiki-redirect-root, rbwiki-redirect-mw",

# attach all middlewares to the router
"traefik.http.routers.rbwiki.middlewares=rbwiki-ratelimit,rbwiki-inflight,rbwiki-redirect-root,rbwiki-redirect-mw",

# redirects
"traefik.http.middlewares.rbwiki-redirect-root.redirectregex.regex=^https://wiki\\.redbrick\\.dcu\\.ie/?$",
"traefik.http.middlewares.rbwiki-redirect-root.redirectregex.replacement=https://wiki.redbrick.dcu.ie/Main_Page",
"traefik.http.middlewares.rbwiki-redirect-mw.redirectregex.regex=https://wiki\\.redbrick\\.dcu\\.ie/Mw/(.*)",
"traefik.http.middlewares.rbwiki-redirect-mw.redirectregex.replacement=https://wiki.redbrick.dcu.ie/$1",

# per-client rate limit
"traefik.http.middlewares.rbwiki-ratelimit.ratelimit.average=20",
"traefik.http.middlewares.rbwiki-ratelimit.ratelimit.period=1s",
"traefik.http.middlewares.rbwiki-ratelimit.ratelimit.burst=40",

# group requests by client IP
"traefik.http.middlewares.rbwiki-ratelimit.ratelimit.sourcecriterion.ipstrategy.depth=1",

# cap concurrent requests per client IP
"traefik.http.middlewares.rbwiki-inflight.inflightreq.amount=10",
"traefik.http.middlewares.rbwiki-inflight.inflightreq.sourcecriterion.ipstrategy.depth=1",
]
}

Expand All @@ -69,52 +85,148 @@ job "mediawiki" {
data = <<EOH
# user www-data www-data;
error_log /dev/stderr error;

events {
worker_connections 1024;
}

http {
include /etc/nginx/mime.types;
server_tokens off;
error_log /dev/stderr error;
access_log /dev/stdout;
charset utf-8;

# Trust only Traefik / ingress
set_real_ip_from 136.206.16.50;
real_ip_header X-Forwarded-For;
real_ip_recursive on;

log_format main '$remote_addr realip=$realip_remote_addr xff="$http_x_forwarded_for" '
'- $remote_user [$time_local] "$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
access_log /dev/stdout main;

# Per-IP limits
limit_conn_zone $binary_remote_addr zone=perip_conn:10m;
limit_req_zone $binary_remote_addr zone=perip_req:10m rate=8r/s;
limit_req_zone $binary_remote_addr zone=perip_php:10m rate=3r/s;

# Known crawler UAs you want to be harsher with
map $http_user_agent $bad_bot {
default 0;
~*(GPTBot|Amazonbot|SEOkicks|meta-externalagent) 1;
}

# Expensive MediaWiki special pages
map $arg_title $is_bad_special {
default 0;
~*^Special:(UserLogin|CreateAccount|RecentChanges|RecentChangesLinked|WhatLinksHere)$ 1;
}

# Expensive actions
map $arg_action $is_bad_action {
default 0;
~*^(edit|history|info|submit)$ 1;
}

# Composite flags
map "$bad_bot:$is_bad_special" $block_bad_special {
default 0;
"1:1" 1;
}

map "$bad_bot:$is_bad_action" $block_bad_action {
default 0;
"1:1" 1;
}

server {
server_name {{ env "NOMAD_META_domain" }};
listen 80;
listen [::]:80;
root /var/www/html;
index index.php index.html index.htm;

client_max_body_size 5m;
client_body_timeout 60;

# MediaWiki short URLs
location / {
try_files $uri $uri/ @rewrite;
}
server_name {{ env "NOMAD_META_domain" }};
listen 80;
listen [::]:80;
root /var/www/html;
index index.php index.html index.htm;

client_max_body_size 5m;
client_body_timeout 60;
keepalive_timeout 15;
send_timeout 30;

limit_conn perip_conn 15;

location / {
limit_req zone=perip_req burst=15 nodelay;
try_files $uri $uri/ @rewrite;
}

location @rewrite {
rewrite ^/(.*)$ /index.php?title=$1&$args;
}
location @rewrite {
limit_req zone=perip_req burst=15 nodelay;
rewrite ^/(.*)$ /index.php?title=$1&$args;
}

location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|otf|eot|ico)$ {
try_files $uri /index.php;
expires max;
log_not_found off;
}
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|otf|eot)$ {
try_files $uri /index.php;
expires max;
log_not_found off;
}

# Pass the PHP scripts to FastCGI server
location ~ \.php$ {
include fastcgi_params;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_pass {{ env "NOMAD_HOST_ADDR_fpm" }};
fastcgi_index index.php;
}
location = /index.php {
limit_req zone=perip_php burst=5 nodelay;

if ($is_bad_special) {
return 429;
}

if ($is_bad_action) {
return 429;
}

if ($block_bad_special) {
return 403;
}

if ($block_bad_action) {
return 403;
}

include fastcgi_params;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param HTTP_X_REAL_IP $remote_addr;
fastcgi_param HTTP_X_FORWARDED_FOR $http_x_forwarded_for;
fastcgi_pass {{ env "NOMAD_HOST_ADDR_fpm" }};
fastcgi_index index.php;

fastcgi_connect_timeout 10s;
fastcgi_send_timeout 30s;
fastcgi_read_timeout 30s;
fastcgi_buffering on;
fastcgi_buffers 16 16k;
fastcgi_buffer_size 32k;
}

location ~ /\.ht {
deny all;
}
location ~ \.php$ {
limit_req zone=perip_php burst=5 nodelay;

include fastcgi_params;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param HTTP_X_REAL_IP $remote_addr;
fastcgi_param HTTP_X_FORWARDED_FOR $http_x_forwarded_for;
fastcgi_pass {{ env "NOMAD_HOST_ADDR_fpm" }};
fastcgi_index index.php;

fastcgi_connect_timeout 10s;
fastcgi_send_timeout 30s;
fastcgi_read_timeout 30s;
fastcgi_buffering on;
fastcgi_buffers 16 16k;
fastcgi_buffer_size 32k;
}

location ~ /\.ht {
deny all;
}
}
}
EOH
Expand Down