Nitter: implemented rate-limiting

config-tasks/nitter.yaml

@@ -7,7 +7,7 @@
 
     - name: Set up nitter conf
       template:
-        src: "conf/nitter.conf.j2"
+        src: "conf/nitter/nitter.conf.j2"
         dest: "{{ docker_dir }}/nitter/nitter.conf"
         owner: 1000
         group: 1000
@@ -29,6 +29,90 @@
         group: 1000
         mode: 0755
 
+# --------------------------------------------------------------------------------------------------
+
+    - name: Create nginx directory
+      file:
+        path: "{{ docker_dir }}/nitter/nginx"
+        state: directory
+        owner: 1000
+        group: 1000
+        mode: 0775
+
+    - name: Touch nitter_error.log
+      file:
+        path: "{{ docker_dir }}/nitter/nginx/nitter_error.log"
+        state: touch
+        owner: 1000
+        group: 1000
+        mode: 0775
+
+    - name: Set up nginx.conf.j2
+      template:
+        src: "conf/nitter/nginx.conf.j2"
+        dest: "{{ docker_dir }}/nitter/nginx/nginx.conf"
+        owner: 1000
+        group: 1000
+        mode: 0755
+
+    - name: Set up nginx
+      template:
+        src: "conf/nitter/nginx.j2"
+        dest: "{{ docker_dir }}/nitter/nginx/nginx"
+        owner: 1000
+        group: 1000
+        mode: 0755
+
+    - name: Create shared_cache.conf
+      copy:
+        dest: "{{ docker_dir }}/nitter/nginx/shared_cache.conf"
+        content: |
+          proxy_buffers 64 16k;
+          proxy_buffer_size 4k;
+          expires 90d;
+          access_log off;
+          resolver 127.0.0.11;
+          set $backend "nitter";
+          proxy_pass http://$backend:8080;
+
+    - name: Create shared_static.conf
+      copy:
+        dest: "{{ docker_dir }}/nitter/nginx/shared_static.conf"
+        content: |
+          expires 90d;
+          access_log off;
+          root /src/public;
+
+    - name: Clone nitter repo to /tmp
+      git:
+        repo: https://github.com/zedeus/nitter.git
+        dest: "/tmp/nitter"
+        single_branch: true
+        version: guest_accounts
+
+    - name: Copy the public folder to the specified directory
+      command: "cp -r /tmp/nitter/public {{ docker_dir }}/nitter/nginx/"
+
+# --------------------------------------------------------------------------------------------------
+
+    - name: Create fail2ban directory
+      file:
+        path: "{{ docker_dir }}/nitter/fail2ban"
+        state: directory
+        owner: 1000
+        group: 1000
+        mode: 0775
+
+    - name: Set up jail.local
+      template:
+        src: "conf/nitter/jail.local.j2"
+        dest: "{{ docker_dir }}/nitter/fail2ban/jail.local"
+        owner: 1000
+        group: 1000
+        mode: 0755
+
+# --------------------------------------------------------------------------------------------------
+
     - name: Curl guest_accounts.jsonl
       shell: curl -s '{{ twitterminator_url }}' > {{ docker_dir }}/nitter/guest_accounts.jsonl
       ignore_errors: true
@@ -37,4 +121,4 @@
       command: chmod 0777 {{ docker_dir }}/nitter/guest_accounts.jsonl
 
     - name: Chown guest_accounts.jsonl
-      command: chown 1000:1000 {{ docker_dir }}/nitter/guest_accounts.jsonl
+      command: chown 1000:1000 {{ docker_dir }}/nitter/guest_accounts.jsonl

templates/compose/nitter.yaml.j2

@@ -4,14 +4,16 @@ services:
   nitter:
     image: git.opnxng.com/opnxng/nitter:latest
     container_name: nitter
+    user: "998:998"
+    read_only: true
     security_opt:
       - no-new-privileges
     environment:
       - PUID=1000
       - PGID=1000
       - TZ=Asia/Singapore
-    ports:
+#    ports:
-      - 8883:8080
+#      - 8883:8080
     volumes:
       - {{ docker_dir }}/nitter/nitter.conf:/src/nitter.conf:ro
       - {{ docker_dir }}/nitter/about.html:/src/public/md/about.html:ro
@@ -39,8 +41,10 @@ services:
       - PGID=1000
       - TZ=Asia/Singapore
     volumes:
-      - {{ docker_dir }}/nitter/nitter-redis:/data
+      - {{ docker_dir }}/nitter/redis:/data
     restart: unless-stopped
+    user: "999:1000"
+    read_only: true
     healthcheck:
       test: redis-cli ping
       interval: 30s
@@ -49,6 +53,49 @@ services:
     networks:
       - nitter
 
+  nitter-nginx:
+    image: nginx:stable-alpine-slim
+    container_name: nitter-nginx
+    security_opt:
+      - no-new-privileges
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=Asia/Singapore
+    volumes:
+      - {{ docker_dir }}/nitter/nginx/nginx.conf:/etc/nginx/nginx.conf
+      - {{ docker_dir }}/nitter/nginx/shared_cache.conf:/etc/nginx/shared_cache.conf
+      - {{ docker_dir }}/nitter/nginx/shared_static.conf:/etc/nginx/shared_static.conf
+      - {{ docker_dir }}/nitter/nginx/nitter_error.log:/var/log/nginx/nitter_error.log
+      - {{ docker_dir }}/nitter/nginx/nginx:/etc/logrotate.d/nginx
+      - {{ docker_dir }}/nitter/nginx/public:/src/public
+      - {{ docker_dir }}/nitter/about.html:/src/public/md/about.html:ro
+      - {{ docker_dir }}/nitter/about.md:/src/public/md/about.md:ro
+    restart: unless-stopped
+    ports:
+      - 8883:80
+    networks:
+      - nitter
+
+  nitter-fail2ban:
+    image: lscr.io/linuxserver/fail2ban:latest
+    container_name: nitter-fail2ban
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+    security_opt:
+      - no-new-privileges
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=Asia/Singapore
+#      - VERBOSITY=-vv
+    volumes:
+      - {{ docker_dir }}/nitter/fail2ban/jail.local:/config/fail2ban/jail.local
+      - {{ docker_dir }}/nitter/nginx/nitter_error.log:/var/log/nginx/nitter_error.log
+    restart: unless-stopped
+    network_mode: host
+
 networks:
   nitter:
     name: nitter

templates/conf/Caddyfile.j2

@@ -188,6 +188,7 @@
 		rate_limit @notstatic {remote.ip} 2r/s 60000 500
 		rate_limit @notstatic {remote.ip} 45r/m 300000 500
 		reverse_proxy {{ oracle4_private_ip }}:8883 {
+			header_up X-Real-IP {remote_host}
 			transport http {compression off}
 		}
 		header {

templates/conf/nitter/jail.local.j2

@@ -0,0 +1,10 @@
+[DEFAULT]
+bantime.increment = true
+
+[nginx-limit-req]
+enabled = true
+port    = http,https
+chain   = DOCKER-USER
+action  = %(known/action)s
+# logpath = %(nginx_error_log)s
+logpath = /var/log/nginx/*.log

templates/conf/nitter/nginx.conf.j2

@@ -0,0 +1,57 @@
+# {{ ansible_managed }}
+events {
+    worker_connections  4096;  ## Default: 1024
+}
+
+http {
+
+  limit_req_zone $binary_remote_addr zone=n.opnxng.com_sec:10m rate=1r/s;
+  limit_req_zone $binary_remote_addr zone=n.opnxng.com_min:10m rate=45r/m;
+
+  server {
+          listen 80 default_server;
+          listen [::]:80 default_server;
+          server_name _;
+
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+          proxy_set_header Host $http_host;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          set_real_ip_from 192.168.0.0/16;
+          real_ip_header X-Forwarded-For;
+          real_ip_recursive on;
+
+          location / {
+            resolver 127.0.0.11;
+            set $backend "nitter";
+            proxy_pass http://$backend:8080;
+            limit_req zone=n.opnxng.com_sec burst=3 nodelay;
+            limit_req zone=n.opnxng.com_min burst=4;
+          }
+
+          location = /robots.txt {
+            add_header Content-Type text/plain;
+            return 200 "User-agent: *\nDisallow: /\n";
+          }
+
+          error_log /var/log/nginx/nitter_error.log notice;
+
+          location /pic/ { include shared_cache.conf; }
+          location /video/ { include shared_cache.conf; }
+
+          # If you are running nitter from docker then change `shared_static.conf` to `shared_cache.conf`
+          location /css/ { include shared_cache.conf; }
+
+          location /js/ { include shared_cache.conf; }
+          location /fonts/ { include shared_static.conf; }
+          location = /apple-touch-icon.png { include shared_static.conf; }
+          location = /apple-touch-icon-precomposed.png { include shared_static.conf; }
+          location = /android-chrome-192x192.png { include shared_static.conf; }
+          location = /favicon-32x32.png { include shared_static.conf; }
+          location = /favicon-16x16.png { include shared_static.conf; }
+          location = /favicon.ico { include shared_static.conf; }
+          location = /logo.png { include shared_static.conf; }
+          location = /site.webmanifest { include shared_static.conf; }
+  }
+}

templates/conf/nitter/nginx.j2

@@ -0,0 +1,30 @@
+/var/log/nginx/*.log {
+	daily
+	missingok
+	rotate 52
+	compress
+	delaycompress
+	notifempty
+	create 640 nginx adm
+	sharedscripts
+	postrotate
+		if [ -f /var/run/nginx.pid ]; then
+			kill -USR1 `cat /var/run/nginx.pid`
+		fi
+	endscript
+}
+
+/var/log/nginx/nitter_error.log {
+	rotate 30
+	size 10M
+	dateext
+	dateformat -%Y-%m-%d
+	missingok
+	compress
+	sharedscripts
+	postrotate
+		if [ -f /var/run/nginx.pid ]; then
+			kill -USR1 `cat /var/run/nginx.pid`
+		fi
+	endscript
+}

vars/services.yaml

@@ -1,12 +1,10 @@
-compose:
+config:
-  oracle1:
-    - proxigram
-  oracle2:
-    - proxigram
-  oracle3:
-    - proxigram
   oracle4:
-    - proxigram
+    - caddy
+
+compose:
+  oracle4:
+    - caddy
 
 # ----------------------------------------------------------------------------------------------------