From a0529dd6a5b70dc262c95d8fc9791eea4d30298b Mon Sep 17 00:00:00 2001 From: Opnxng Date: Fri, 8 Dec 2023 09:12:57 +0800 Subject: [PATCH] Nitter: implemented rate-limiting --- config-tasks/nitter.yaml | 88 ++++++++++++++++++- templates/compose/nitter.yaml.j2 | 53 ++++++++++- templates/conf/Caddyfile.j2 | 1 + ...{nginx.conf.j2 => invidious-nginx.conf.j2} | 0 templates/conf/nitter/jail.local.j2 | 10 +++ templates/conf/nitter/nginx.conf.j2 | 57 ++++++++++++ templates/conf/nitter/nginx.j2 | 30 +++++++ templates/conf/{ => nitter}/nitter.conf.j2 | 0 vars/services.yaml | 14 ++- 9 files changed, 240 insertions(+), 13 deletions(-) rename templates/conf/{nginx.conf.j2 => invidious-nginx.conf.j2} (100%) create mode 100755 templates/conf/nitter/jail.local.j2 create mode 100644 templates/conf/nitter/nginx.conf.j2 create mode 100644 templates/conf/nitter/nginx.j2 rename templates/conf/{ => nitter}/nitter.conf.j2 (100%) diff --git a/config-tasks/nitter.yaml b/config-tasks/nitter.yaml index e46a68f..0d171b3 100644 --- a/config-tasks/nitter.yaml +++ b/config-tasks/nitter.yaml @@ -7,7 +7,7 @@ - name: Set up nitter conf template: - src: "conf/nitter.conf.j2" + src: "conf/nitter/nitter.conf.j2" dest: "{{ docker_dir }}/nitter/nitter.conf" owner: 1000 group: 1000 @@ -29,6 +29,90 @@ group: 1000 mode: 0755 +# -------------------------------------------------------------------------------------------------- + + - name: Create nginx directory + file: + path: "{{ docker_dir }}/nitter/nginx" + state: directory + owner: 1000 + group: 1000 + mode: 0775 + + - name: Touch nitter_error.log + file: + path: "{{ docker_dir }}/nitter/nginx/nitter_error.log" + state: touch + owner: 1000 + group: 1000 + mode: 0775 + + - name: Set up nginx.conf.j2 + template: + src: "conf/nitter/nginx.conf.j2" + dest: "{{ docker_dir }}/nitter/nginx/nginx.conf" + owner: 1000 + group: 1000 + mode: 0755 + + - name: Set up nginx + template: + src: "conf/nitter/nginx.j2" + dest: "{{ docker_dir }}/nitter/nginx/nginx" + owner: 1000 + group: 1000 + mode: 0755 + + - name: Create shared_cache.conf + copy: + dest: "{{ docker_dir }}/nitter/nginx/shared_cache.conf" + content: | + proxy_buffers 64 16k; + proxy_buffer_size 4k; + expires 90d; + access_log off; + resolver 127.0.0.11; + set $backend "nitter"; + proxy_pass http://$backend:8080; + + - name: Create shared_static.conf + copy: + dest: "{{ docker_dir }}/nitter/nginx/shared_static.conf" + content: | + expires 90d; + access_log off; + root /src/public; + + - name: Clone nitter repo to /tmp + git: + repo: https://github.com/zedeus/nitter.git + dest: "/tmp/nitter" + single_branch: true + version: guest_accounts + + - name: Copy the public folder to the specified directory + command: "cp -r /tmp/nitter/public {{ docker_dir }}/nitter/nginx/" + +# -------------------------------------------------------------------------------------------------- + + - name: Create fail2ban directory + file: + path: "{{ docker_dir }}/nitter/fail2ban" + state: directory + owner: 1000 + group: 1000 + mode: 0775 + + - name: Set up jail.local + template: + src: "conf/nitter/jail.local.j2" + dest: "{{ docker_dir }}/nitter/fail2ban/jail.local" + owner: 1000 + group: 1000 + mode: 0755 + +# -------------------------------------------------------------------------------------------------- + - name: Curl guest_accounts.jsonl shell: curl -s '{{ twitterminator_url }}' > {{ docker_dir }}/nitter/guest_accounts.jsonl ignore_errors: true @@ -37,4 +121,4 @@ command: chmod 0777 {{ docker_dir }}/nitter/guest_accounts.jsonl - name: Chown guest_accounts.jsonl - command: chown 1000:1000 {{ docker_dir }}/nitter/guest_accounts.jsonl \ No newline at end of file + command: chown 1000:1000 {{ docker_dir }}/nitter/guest_accounts.jsonl diff --git a/templates/compose/nitter.yaml.j2 b/templates/compose/nitter.yaml.j2 index 0aab7f7..8548719 100644 --- a/templates/compose/nitter.yaml.j2 +++ b/templates/compose/nitter.yaml.j2 @@ -4,14 +4,16 @@ services: nitter: image: git.opnxng.com/opnxng/nitter:latest container_name: nitter + user: "998:998" + read_only: true security_opt: - no-new-privileges environment: - PUID=1000 - PGID=1000 - TZ=Asia/Singapore - ports: - - 8883:8080 +# ports: +# - 8883:8080 volumes: - {{ docker_dir }}/nitter/nitter.conf:/src/nitter.conf:ro - {{ docker_dir }}/nitter/about.html:/src/public/md/about.html:ro @@ -39,8 +41,10 @@ services: - PGID=1000 - TZ=Asia/Singapore volumes: - - {{ docker_dir }}/nitter/nitter-redis:/data + - {{ docker_dir }}/nitter/redis:/data restart: unless-stopped + user: "999:1000" + read_only: true healthcheck: test: redis-cli ping interval: 30s @@ -49,6 +53,49 @@ services: networks: - nitter + nitter-nginx: + image: nginx:stable-alpine-slim + container_name: nitter-nginx + security_opt: + - no-new-privileges + environment: + - PUID=1000 + - PGID=1000 + - TZ=Asia/Singapore + volumes: + - {{ docker_dir }}/nitter/nginx/nginx.conf:/etc/nginx/nginx.conf + - {{ docker_dir }}/nitter/nginx/shared_cache.conf:/etc/nginx/shared_cache.conf + - {{ docker_dir }}/nitter/nginx/shared_static.conf:/etc/nginx/shared_static.conf + - {{ docker_dir }}/nitter/nginx/nitter_error.log:/var/log/nginx/nitter_error.log + - {{ docker_dir }}/nitter/nginx/nginx:/etc/logrotate.d/nginx + - {{ docker_dir }}/nitter/nginx/public:/src/public + - {{ docker_dir }}/nitter/about.html:/src/public/md/about.html:ro + - {{ docker_dir }}/nitter/about.md:/src/public/md/about.md:ro + restart: unless-stopped + ports: + - 8883:80 + networks: + - nitter + + nitter-fail2ban: + image: lscr.io/linuxserver/fail2ban:latest + container_name: nitter-fail2ban + cap_add: + - NET_ADMIN + - NET_RAW + security_opt: + - no-new-privileges + environment: + - PUID=1000 + - PGID=1000 + - TZ=Asia/Singapore +# - VERBOSITY=-vv + volumes: + - {{ docker_dir }}/nitter/fail2ban/jail.local:/config/fail2ban/jail.local + - {{ docker_dir }}/nitter/nginx/nitter_error.log:/var/log/nginx/nitter_error.log + restart: unless-stopped + network_mode: host + networks: nitter: name: nitter diff --git a/templates/conf/Caddyfile.j2 b/templates/conf/Caddyfile.j2 index caaa5ad..8ca6a52 100755 --- a/templates/conf/Caddyfile.j2 +++ b/templates/conf/Caddyfile.j2 @@ -188,6 +188,7 @@ rate_limit @notstatic {remote.ip} 2r/s 60000 500 rate_limit @notstatic {remote.ip} 45r/m 300000 500 reverse_proxy {{ oracle4_private_ip }}:8883 { + header_up X-Real-IP {remote_host} transport http {compression off} } header { diff --git a/templates/conf/nginx.conf.j2 b/templates/conf/invidious-nginx.conf.j2 similarity index 100% rename from templates/conf/nginx.conf.j2 rename to templates/conf/invidious-nginx.conf.j2 diff --git a/templates/conf/nitter/jail.local.j2 b/templates/conf/nitter/jail.local.j2 new file mode 100755 index 0000000..20d4ed1 --- /dev/null +++ b/templates/conf/nitter/jail.local.j2 @@ -0,0 +1,10 @@ +[DEFAULT] +bantime.increment = true + +[nginx-limit-req] +enabled = true +port = http,https +chain = DOCKER-USER +action = %(known/action)s +# logpath = %(nginx_error_log)s +logpath = /var/log/nginx/*.log diff --git a/templates/conf/nitter/nginx.conf.j2 b/templates/conf/nitter/nginx.conf.j2 new file mode 100644 index 0000000..c69da31 --- /dev/null +++ b/templates/conf/nitter/nginx.conf.j2 @@ -0,0 +1,57 @@ +# {{ ansible_managed }} +events { + worker_connections 4096; ## Default: 1024 +} + +http { + + limit_req_zone $binary_remote_addr zone=n.opnxng.com_sec:10m rate=1r/s; + limit_req_zone $binary_remote_addr zone=n.opnxng.com_min:10m rate=45r/m; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + set_real_ip_from 192.168.0.0/16; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + + location / { + resolver 127.0.0.11; + set $backend "nitter"; + proxy_pass http://$backend:8080; + limit_req zone=n.opnxng.com_sec burst=3 nodelay; + limit_req zone=n.opnxng.com_min burst=4; + } + + location = /robots.txt { + add_header Content-Type text/plain; + return 200 "User-agent: *\nDisallow: /\n"; + } + + error_log /var/log/nginx/nitter_error.log notice; + + location /pic/ { include shared_cache.conf; } + location /video/ { include shared_cache.conf; } + + # If you are running nitter from docker then change `shared_static.conf` to `shared_cache.conf` + location /css/ { include shared_cache.conf; } + + location /js/ { include shared_cache.conf; } + location /fonts/ { include shared_static.conf; } + location = /apple-touch-icon.png { include shared_static.conf; } + location = /apple-touch-icon-precomposed.png { include shared_static.conf; } + location = /android-chrome-192x192.png { include shared_static.conf; } + location = /favicon-32x32.png { include shared_static.conf; } + location = /favicon-16x16.png { include shared_static.conf; } + location = /favicon.ico { include shared_static.conf; } + location = /logo.png { include shared_static.conf; } + location = /site.webmanifest { include shared_static.conf; } + } +} \ No newline at end of file diff --git a/templates/conf/nitter/nginx.j2 b/templates/conf/nitter/nginx.j2 new file mode 100644 index 0000000..cadea26 --- /dev/null +++ b/templates/conf/nitter/nginx.j2 @@ -0,0 +1,30 @@ +/var/log/nginx/*.log { + daily + missingok + rotate 52 + compress + delaycompress + notifempty + create 640 nginx adm + sharedscripts + postrotate + if [ -f /var/run/nginx.pid ]; then + kill -USR1 `cat /var/run/nginx.pid` + fi + endscript +} + +/var/log/nginx/nitter_error.log { + rotate 30 + size 10M + dateext + dateformat -%Y-%m-%d + missingok + compress + sharedscripts + postrotate + if [ -f /var/run/nginx.pid ]; then + kill -USR1 `cat /var/run/nginx.pid` + fi + endscript +} diff --git a/templates/conf/nitter.conf.j2 b/templates/conf/nitter/nitter.conf.j2 similarity index 100% rename from templates/conf/nitter.conf.j2 rename to templates/conf/nitter/nitter.conf.j2 diff --git a/vars/services.yaml b/vars/services.yaml index f0213b4..8128eeb 100644 --- a/vars/services.yaml +++ b/vars/services.yaml @@ -1,12 +1,10 @@ -compose: - oracle1: - - proxigram - oracle2: - - proxigram - oracle3: - - proxigram +config: oracle4: - - proxigram + - caddy + +compose: + oracle4: + - caddy # ----------------------------------------------------------------------------------------------------