Websocket request blocked "502 Bad Gateway"

  • Used Zammad version: 6.4.1
  • Used Zammad installation type: docker-compose
  • Operating system: Debian 12
  • Browser + version: Affects all browsers

Expected behavior:

Websocket connection working without errors.

Actual behavior:

Websocket requests blocked with “502 Bad Gateway”. Additional errors: NS_ERROR_NET_TIMEOUT, NS_BINDING_ABORTED, NS_ERROR_WEBSOCKET_CONNECTION_REFUSED.

Suspecting this to be the root cause of the list of recent tickets being very slow to load, but otherwise not seeing any obvious problems in Zammad caused by this issue.
Screendump from browser console:
image

My docker-compose.yml:

---
version: "3.8"

x-shared:
  zammad-service: &zammad-service
    environment: &zammad-environment
      MEMCACHE_SERVERS: ${MEMCACHE_SERVERS:-zammad-memcached:11211}
      POSTGRESQL_DB: ${POSTGRES_DB:-zammad_production}
      POSTGRESQL_HOST: ${POSTGRES_HOST:-zammad-postgresql}
      POSTGRESQL_USER: ${POSTGRES_USER:-zammad}
      POSTGRESQL_PASS: ${POSTGRES_PASS:-zammad}
      POSTGRESQL_PORT: ${POSTGRES_PORT:-5432}
      POSTGRESQL_OPTIONS: ${POSTGRESQL_OPTIONS:-?pool=50}
      POSTGRESQL_DB_CREATE:
      REDIS_URL: ${REDIS_URL:-redis://zammad-redis:6379}
      S3_URL:
      # Backup settings
      BACKUP_DIR: "${BACKUP_DIR:-/var/tmp/zammad}"
      BACKUP_TIME: "${BACKUP_TIME:-03:00}"
      HOLD_DAYS: "${HOLD_DAYS:-10}"
      TZ: "${TZ:-Europe/Berlin}"
      # Allow passing in these variables via .env:
      AUTOWIZARD_JSON:
      AUTOWIZARD_RELATIVE_PATH:
      ELASTICSEARCH_ENABLED:
      ELASTICSEARCH_SCHEMA:
      ELASTICSEARCH_HOST:
      ELASTICSEARCH_PORT:
      ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-elastic}
      ELASTICSEARCH_PASS: ${ELASTICSEARCH_PASS:-zammad}
      ELASTICSEARCH_NAMESPACE:
      ELASTICSEARCH_REINDEX:
      ELASTICSEARCH_SSL_VERIFY:
      NGINX_PORT:
      NGINX_CLIENT_MAX_BODY_SIZE:
      NGINX_SERVER_NAME:
      NGINX_SERVER_SCHEME:
      RAILS_TRUSTED_PROXIES:
      ZAMMAD_HTTP_TYPE:
      ZAMMAD_FQDN:
      ZAMMAD_WEB_CONCURRENCY:
      ZAMMAD_SESSION_JOBS_CONCURRENT:
      ZAMMAD_PROCESS_SCHEDULED_JOBS_WORKERS:
      ZAMMAD_PROCESS_DELAYED_JOBS_WORKERS:
      # Variables used by ngingx-proxy container for reverse proxy creations
      # for docs refer to https://github.com/nginx-proxy/nginx-proxy
      VIRTUAL_HOST:
      VIRTUAL_PORT:
      # Variables used by acme-companion for retrieval of LetsEncrypt certificate
      # for docs refer to https://github.com/nginx-proxy/acme-companion
      LETSENCRYPT_HOST:
      LETSENCRYPT_EMAIL:

    image: ${IMAGE_REPO:-ghcr.io/zammad/zammad}:${VERSION:-6.4.1-64}
    restart: ${RESTART:-always}
    volumes:
      - /var/opt/zammad/storage:/opt/zammad/storage
    depends_on:
      - zammad-memcached
      - zammad-postgresql
      - zammad-redis

services:
  zammad-backup:
    <<: *zammad-service
    command: ["zammad-backup"]
    volumes:
      - /var/opt/zammad/backup:/var/tmp/zammad
      - /var/opt/zammad/storage:/opt/zammad/storage:ro
    user: 0:0

  zammad-elasticsearch:
    image: bitnami/elasticsearch:${ELASTICSEARCH_VERSION:-8.17.3}
    restart: ${RESTART:-always}
    volumes:
      - /var/opt/zammad/elasticsearch:/bitnami/elasticsearch/data
    environment:
      # Enable authorization without HTTPS. For external access with
      #   SSL termination, use solutions like nginx-proxy-manager.
      ELASTICSEARCH_ENABLE_SECURITY: 'true'
      ELASTICSEARCH_SKIP_TRANSPORT_TLS: 'true'
      ELASTICSEARCH_ENABLE_REST_TLS: 'false'
      # ELASTICSEARCH_USER is hardcoded to 'elastic' in the container.
      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASS:-zammad}

  zammad-init:
    <<: *zammad-service
    command: ["zammad-init"]
    depends_on:
      - zammad-postgresql
    restart: on-failure
    user: 0:0

  zammad-memcached:
    command: memcached -m 256M
    image: memcached:${MEMCACHE_VERSION:-1.6.37-alpine}
    restart: ${RESTART:-always}

  zammad-nginx:
    <<: *zammad-service
    command: ["zammad-nginx"]
    networks:
      - default
      - zammad-nginx-external-network
    ports: []
    depends_on:
      - zammad-railsserver

  zammad-postgresql:
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-zammad_production}
      POSTGRES_USER: ${POSTGRES_USER:-zammad}
      POSTGRES_PASSWORD: ${POSTGRES_PASS:-zammad}
    image: postgres:${POSTGRES_VERSION:-17.4-alpine}
    restart: ${RESTART:-always}
    volumes:
      - /var/opt/zammad/postgresql:/var/lib/postgresql/data

  zammad-railsserver:
    <<: *zammad-service
    command: ["zammad-railsserver"]

  zammad-redis:
    image: redis:${REDIS_VERSION:-7.4.2-alpine}
    restart: ${RESTART:-always}
    volumes:
      - /var/opt/zammad/redis:/data

  zammad-scheduler:
    <<: *zammad-service
    command: ["zammad-scheduler"]

  zammad-websocket:
    <<: *zammad-service
    command: ["zammad-websocket"]

networks:
  zammad-nginx-external-network:
    external: true
    name: ${ZAMMAD_NGINX_EXTERNAL_NETWORK}

And my .env:

# General
TZ=Europe/Copenhagen
ZAMMAD_FQDN=zammad.redacted.hosting
VERSION=latest
#ZAMMAD_HTTP_TYPE=https  # Use HTTPS if behind a reverse proxy

# Setting secure passwords
POSTGRES_USER=zammad
POSTGRES_PASS=redacted
POSTGRES_DB=zammad-postgresql
ELASTICSEARCH_PASS=redacted

# NGINX
ZAMMAD_NGINX_EXTERNAL_NETWORK=zammad-net
NGINX_EXPOSE_PORT=8080  # Publicly exposed Zammad port
NGINX_CLIENT_MAX_BODY_SIZE=50M  # Allow larger uploads
#NGINX_SERVER_NAME=zammad.redacted.hosting # make nginx serve only this site

#ZAMMAD_WEBSOCKET_PORT=6042
#ZAMMAD_WEBSOCKET_HOST=zammad-websocket

I then have an nginx proxy in front of the Zammad stack, with this docker-compose.yml:

services:
  nginx:
    image: nginx:latest
    container_name: https_proxy
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - /opt/docker/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
      - /opt/docker/nginx/cert/fullchain.pem:/etc/nginx/certs/fullchain.pem:ro
      - /opt/docker/nginx/cert/key.pem:/etc/nginx/certs/privkey.pem:ro
    networks:
      - zammad-net
    environment:
      - TZ=Europe/Copenhagen

networks:
  zammad-net:
    external: true

And this conf:

worker_processes auto;

events {
}

http {

    server {

    	    listen 443 ssl;
	    http2 on;
	    server_name zammad.redacted.hosting;

	    ssl_certificate /etc/nginx/certs/fullchain.pem;
	    ssl_certificate_key /etc/nginx/certs/privkey.pem;

	    ssl_protocols TLSv1.2 TLSv1.3;
	    ssl_prefer_server_ciphers on;
	    ssl_ciphers "HIGH:EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:!aNULL:!MD5:!RC4";
	    ssl_session_timeout 1d;
	    ssl_session_tickets off;

	    add_header Strict-Transport-Security "max-age=3600; includeSubDomains; preload" always;
	    add_header X-Frame-Options SAMEORIGIN;
	    add_header X-XSS-Protection "1; mode=block";
	    add_header X-Content-Type-Options nosniff;
	    add_header Referrer-Policy "no-referrer-when-downgrade";
	    server_tokens off;

	    if ($host !~* ^(zammad\.redacted\.hosting)$) {
		return 444;
	    }
	    if ($request_method ~ ^(TRACE)$ ) {
		return 405;
	    }
	    client_max_body_size 50M;
	    location / {
		proxy_pass http://zammad-nginx:8080; # Connect to the existing Zammad service
		proxy_set_header Host $host;
		proxy_set_header X-Real-IP $remote_addr;
		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
		proxy_set_header X-Forwarded-Proto https;
		proxy_set_header X-Forwarded-Ssl on;
            }

    	}

    }

I did try adding a second “/ws” location to the external nginx proxy, and proxying this through to the zammad-websocket container, but this only caused the nginx container to fail due to being “unable to find the upstream resource” or something of that sort.
I tried this approach based on this community post: Recommended docker compose + nginx setup /w websocket support - #2 by Seneral

I’m happy to provide any additional info needed for troubleshooting - just ask :slight_smile:

Steps to reproduce the behavior:

Just refreshing the Zammad page will throw errors in browser console.
To be clear, Zammad does actually work in general terms, just want to get rid of the websocket errors and optimise loading of recent tickets list, if possible.

Possibly relevant detail:

When I look at current sessions in Zammad, they all show with the same IP address of 172.20.0.3.
So for some reason, Zammad is not seeing the actual client - not sure if this could play a role in relation to the websockets issue or not…