EFW Support

Support => EFW SMTP, HTTP, SIP, FTP Proxy Support => Topic started by: beto2p on Friday 13 July 2018, 12:01:10 am


Title: Consumo alto de memória Squid Proxy HTTPS
Post by: beto2p on Friday 13 July 2018, 12:01:10 am
Estou utilizando o Endian 3.2.5 com o Proxy Transparente HTTP + Webfilter.
Estava rodando tudo normalmente por vários meses.
Recentemente ativei o Proxy HTTPS, e após a ativação o consumo de memória do Squid ficou muito alto, chegando a travar o serviço umas duas vezes por dia. Quando reinicio o Squid o consumo volta ao normal, mas ele vai aumentando de forma crescente até chegar no limite do servidor.
Estou usando um servidor Itautec com Intel Xeon e 4GB de memória.
Mesmo fora do horário de uso da empresa onde somente alguns servidores ficam ligados o consumo de memória vai aumentado de forma crescente.
Já alterei as configurações do cache do Squid para valores bem baixos, já desativei os logs do Proxy e o clamAV, mas não resolveu.

Lembrando que o problema acorreu só após a ativação do proxy HTTPS.

Segue as configurações do squid.conf

Code:
shutdown_lifetime 1 seconds
icp_port 0

workers 1

# direct access - acls
acl to_proxy_port           port 8080 18080 18081
# proxy interfaces - acls
acl to_green_interface    dst 10.1.1.1

acl from_green          src "/etc/squid/acls/green_subnets.acl"
acl to_green            dst "/etc/squid/acls/green_subnets.acl"

tcp_outgoing_mark 0x20000000
tcp_preserve_outgoing_mark_mask 0x3fff8

#=== GREEN zone setting ===
#=== GREEN IP 10.1.1.1 ===
http_port 10.1.1.1:8080 ssl-bump cert=/var/efw/proxy/https_cert generate-host-certificates=on cipher=ALL:!ADH:!EXP:!eNULL:!aNULL:!SSLv2:!RC4:!LOW:!MD5:!DES options=NO_SSLv2,NO_SSLv3
http_port 10.1.1.1:18080 intercept
https_port 10.1.1.1:18081 intercept ssl-bump cert=/var/efw/proxy/https_cert generate-host-certificates=on cipher=ALL:!ADH:!EXP:!eNULL:!aNULL:!SSLv2:!RC4:!LOW:!MD5:!DES options=NO_SSLv2,NO_SSLv3


acl bypass_host_strict_check_acl ssl::server_name_regex .*
bypass_host_strict_check allow bypass_host_strict_check_acl
ssl_bump splice localhost
ssl_bump splice to_proxy_port
acl bypass_windows ssl::server_name "/etc/squid/acls/https_bypass_rules.acl"
ssl_bump splice bypass_windows
acl BrokenButTrustedServers dstdomain "/etc/squid/acls/https_bypass_dstdom_broken.acl"
acl DomainMismatch ssl_error SQUID_X509_V_ERR_DOMAIN_MISMATCH
sslproxy_cert_error allow BrokenButTrustedServers DomainMismatch
acl ssl_step1 at_step SslBump1
acl ssl_step2 at_step SslBump2
ssl_bump peek ssl_step1
ssl_bump bump all
acl https_proto proto https
always_direct allow https_proto
sslproxy_cert_error allow all
sslproxy_flags DONT_VERIFY_PEER
sslproxy_cert_sign_hash sha256


dns_v4_first on

cache_effective_user squid

pid_filename /var/run/squid.pid

cache_mem 100 MB

cache_dir rock /var/spool/squid 2000 max-size=1048576

error_directory /usr/share/squid/errors/en

icon_directory /usr/share/squid/icons

max_filedesc 100415

server_persistent_connections off
half_closed_clients off
buffered_logs on

# START LOG
cache_log /dev/null
cache_access_log /dev/null
cache_store_log none

log_mime_hdrs off
# END LOG

# FORWARD IP ADDRESS
forwarded_for delete

# START AUTHENTICATION
# METHOD is NCSA
auth_param basic program /usr/lib/squid/basic_ncsa_auth /var/efw/proxy/ncsausers
auth_param basic children 20
auth_param basic realm Proxy Server
auth_param basic credentialsttl 60 minutes
    
acl for_auth_users proxy_auth REQUIRED
# END AUTHENTICATION

# network - acls
acl from_all                src all
acl to_all                  dst all

acl from_localhost          src 127.0.0.1/32
acl to_localhost            dst 127.0.0.1/32
acl CONNECT                 method CONNECT

acl to_http_port            port 80
acl to_https_port           port 10443

# allowed ports - acls
acl allowed_ports       port "/etc/squid/acls/ports.acl"
acl allowed_sslports    port "/etc/squid/acls/sslports.acl"


acl from_rule0 arp "/etc/squid/acls/src_rule0.acl"
acl within_timeframe_rule0 time MTWHFAS 00:00-24:00
acl from_rule1 arp "/etc/squid/acls/src_rule1.acl"
acl within_timeframe_rule1 time MTWHFAS 00:00-24:00
acl within_timeframe_rule2 time MTWHFAS 00:00-24:00

# caching settings
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
refresh_pattern .            0 20% 4320

cache deny      from_localhost
cache deny      CONNECT
cache allow     from_all

# http access to cachemanager
acl cachemanageracl proto cache_object
http_access allow cachemanageracl from_localhost
http_access deny cachemanageracl

# snmp access settings
snmp_port 3401
acl snmppublic snmp_community public
snmp_access allow snmppublic from_localhost
snmp_access deny from_all

# http access to squid
http_access deny    to_localhost
http_access allow   from_localhost
http_access allow   from_green to_green_interface to_http_port
http_access allow   from_green to_green_interface to_https_port
http_access allow   CONNECT from_green to_green_interface to_https_port
http_access deny    to_green_interface to_https_port
http_access deny    to_green_interface to_proxy_port

http_access deny    !allowed_ports !allowed_sslports
http_access deny    CONNECT !allowed_sslports

http_access allow from_rule0  within_timeframe_rule0    
http_access allow from_rule1  within_timeframe_rule1    
http_access allow   within_timeframe_rule2    
http_access deny    from_all

# http reply access rules
http_reply_access allow from_localhost
http_reply_access allow from_rule0  within_timeframe_rule0    
http_reply_access allow from_rule1  within_timeframe_rule1    
http_reply_access allow   within_timeframe_rule2    
http_reply_access deny from_all

# max/min object size
maximum_object_size 1024 KB
minimum_object_size 0 KB

visible_hostname efw01.copal.local

# begin custom.tmpl
# end custom.tmpl

icap_enable on
icap_service_revival_delay 30
icap_service_failure_limit -1
icap_preview_enable on
icap_preview_size    128
icap_send_client_ip  on
icap_send_client_username  on

include /etc/squid/squid.conf.d/*.conf

adaptation_access service_cf_req deny cachemanageracl

# icap contentfilter access control
# rule 0 - none
adaptation_access service_cf_req deny from_rule0  within_timeframe_rule0   
# rule 1 - bloqueio_parcial
adaptation_access service_cf_req allow !CONNECT from_rule1  within_timeframe_rule1   
adaptation_access service_cf_req allow CONNECT ssl_step2 from_rule1  within_timeframe_rule1   
adaptation_meta X-Profile profilebloqueio_parcial from_rule1  within_timeframe_rule1   
# rule 2 - bloqueio_paginas
adaptation_access service_cf_req allow !CONNECT   within_timeframe_rule2   
adaptation_access service_cf_req allow CONNECT ssl_step2   within_timeframe_rule2   
adaptation_meta X-Profile profilebloqueio_paginas   within_timeframe_rule2   
# default deny - only allow defined traffic
adaptation_access service_cf_req deny all



Title: Re: Consumo alto de memória Squid Proxy HTTPS
Post by: Dark-Vex on Monday 16 July 2018, 06:34:29 pm
Hi, how many users do you have behind this system?

Title: Re: Consumo alto de memória Squid Proxy HTTPS
Post by: beto2p on Saturday 21 July 2018, 07:19:18 am
Quote from: Dark-Vex on Monday 16 July  2018, 06:34:29 pm
Hi, how many users do you have behind this system?

80 users.

I've upgraded memory to 8GB. But it did not solve the problem.
Memory consumption continues to increase gradually until it reaches the limit.
I scheduled in crontab daily tasks to restart squid. But this is a temporary solution

Title: Re: Consumo alto de memória Squid Proxy HTTPS
Post by: Dark-Vex on Monday 23 July 2018, 05:39:04 pm
The memory continues to increase because squid cache all the default CA certificates (they are ~600 default CA).
On the squid forum they suggest to add the parameter "sslflags=NO_DEFAULT_CA" in order to don't cache the CA.
So if you can edit /etc/squid/squid.conf.tmpl and where see these lines

Code:
    #if $HTTPS_MODE != 'disabled' and $HTTPS_CERT
http_port $ip_addr:$PROXY_PORT ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS
      #if $transparent or $tproxy
http_port $ip_addr:18080 $intercept_kind
https_port $ip_addr:18081 $intercept_kind ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS
      #end if


please "sslflags=NO_DEFAULT_CA" like this:
Code:
    #if $HTTPS_MODE != 'disabled' and $HTTPS_CERT
http_port $ip_addr:$PROXY_PORT ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS sslflags=NO_DEFAULT_CA
      #if $transparent or $tproxy
http_port $ip_addr:18080 $intercept_kind
https_port $ip_addr:18081 $intercept_kind ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS sslflags=NO_DEFAULT_CA
      #end if

then save the file and restart squid with "jobcontrol restart squid --force"

Title: Re: Consumo alto de memória Squid Proxy HTTPS [SOLVED]
Post by: beto2p on Wednesday 25 July 2018, 10:23:47 pm
Adding the parameter "sslflags = NO_DEFAULT_CA" has solved the problem.
Memory consumption has stabilized.
Thank you very much.

Quote from: Dark-Vex on Monday 23 July  2018, 05:39:04 pm
The memory continues to increase because squid cache all the default CA certificates (they are ~600 default CA).
On the squid forum they suggest to add the parameter "sslflags=NO_DEFAULT_CA" in order to don't cache the CA.
So if you can edit /etc/squid/squid.conf.tmpl and where see these lines

Code:
    #if $HTTPS_MODE != 'disabled' and $HTTPS_CERT
http_port $ip_addr:$PROXY_PORT ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS
      #if $transparent or $tproxy
http_port $ip_addr:18080 $intercept_kind
https_port $ip_addr:18081 $intercept_kind ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS
      #end if


please "sslflags=NO_DEFAULT_CA" like this:
Code:
    #if $HTTPS_MODE != 'disabled' and $HTTPS_CERT
http_port $ip_addr:$PROXY_PORT ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS sslflags=NO_DEFAULT_CA
      #if $transparent or $tproxy
http_port $ip_addr:18080 $intercept_kind
https_port $ip_addr:18081 $intercept_kind ssl-bump cert=$HTTPS_CERT generate-host-certificates=on cipher=$HTTPS_CIPHERS options=$HTTPS_OPTIONS sslflags=NO_DEFAULT_CA
      #end if

then save the file and restart squid with "jobcontrol restart squid --force"



Powered by SMF 1.1 RC2 | SMF © 2001-2005, Lewis Media