Extracting some HTTP

The following LUA script is a little more complex, we will now extract HTTP request and response from our capture file and split them per TCP streams.

Note

You can use the 0.cap previously created as source to try out your code.:

tshark -X lua_script:http.lua -r streams/0.cap -q

Using other streams is unlikely to work, try to workout why.

After execution, you should have a whole bunch of S-<id>.txt in your streams folder

Script:

-- Let's register all the fields we want TShark to extract
-- Remember that we are creating functions retrieving the field values.

local tcp_stream = Field.new("tcp.stream")
local http_method = Field.new("http.request.method")
local http_uri = Field.new("http.request.uri")
local http_version = Field.new("http.request.version")
local http_code = Field.new("http.response.code")
local http_phrase = Field.new("http.response.phrase")
local http_location = Field.new("http.location")
local http_request = Field.new("http.request")
local http_response = Field.new("http.response")
local http_cookie = Field.new("http.cookie_pair")
local http_setcookie = Field.new("http.set_cookie")
local http_request_header = Field.new("http.request.line")
local http_response_header = Field.new("http.response.line")
local http_response_data = Field.new("http.file_data")

-- Creating the listener, to avoid http data appearing twice due to retransmit, let's suppress them
local tap = Listener.new(nil,"http && !tcp.analysis.retransmission && !tcp.analysis.lost_segment")

local dumpers = {}

-- To avoid runtime error due to nil variable, this function will simply display nothing if TShark has not found the field we are looking for.
local function to_string(string)
        if (string == nil) then
                return ""
        else
                return tostring(string)
        end
end

-- Same as the previous example, but this time we are writing text with normal LUA I/O functions.
local function write_msg(id,msg)
        local file = dumpers[id]
        if not file then
                file = io.open("streams/" .. id .. ".txt", "a")
                if (file == nil) then
                        for file,dumper in pairs(dumpers) do
                                dumper:flush()
                                dumper:close()
                        end
                        dumpers = {}
                        file = assert(io.open("streams/" .. id .. ".txt", "a"))
                end
                dumpers[id] = file
        end
        file:write(msg)
end

function tap.packet(pinfo,tvb,tapdata)

        -- The only reason I am checking if this is a http request is due to the cookies
        if ( http_request() ) then
                if( http_method() == nil ) then return end

                local request_mrhsession = ""
                local cookie = {http_cookie()}
                for i in pairs(cookie) do
                        -- One of the powerful function of LUA is its text matching
                        request_mrhsession = tostring(cookie[i]):match '.*LastMRH_Session=(%w+).*'
                        if (request_mrhsession ~= nil ) then break end
                end

                -- pinfo is from our tap, and contains packet information created by Wireshark, like timestamps.
                local msg =
                        to_string("** " .. string.format("%8d", pinfo.number)) .. " ** " ..
                        to_string(format_date(pinfo.abs_ts)) .. " " ..
                        to_string(http_method()) .. " " ..
                        to_string(http_uri()) .. " " ..
                        to_string(http_version()) .. " " ..
                        to_string(request_mrhsession) .. " " ..
                        to_string(tcp_stream())
                        .. "\n"

                -- Writing to file all of our HTTP request headers, we are adding a "S-" prefix to the filename
                write_msg("S-" .. tostring(tcp_stream()),msg)
                local header = {http_request_header()}
                for i in pairs(header) do
                        write_msg("S-" .. tostring(tcp_stream()),"               " .. tostring(header[i]))
                end

        else

                -- If that is not a request, then it is a response :)
                local response_mrhsession = ""
                local cookie = {http_setcookie()}
                for i in pairs(cookie) do
                        response_mrhsession = tostring(cookie[i]):match '.*LastMRH_Session=(%w+).*'
                        if (response_mrhsession ~= nil ) then break end
                end

                local msg =
                        to_string("** " .. string.format("%8d", pinfo.number)) .. " ** " ..
                        to_string(format_date(pinfo.abs_ts)) .. " " ..
                        to_string(http_code()) .. " " ..
                        to_string(http_phrase()) .. " " ..
                        to_string(http_version()) .. " " ..
                        to_string(response_mrhsession) .. " " ..
                        to_string(http_location()) .. " " ..
                        to_string(tcp_stream())
                        .. "\n"

                write_msg("S-" .. tostring(tcp_stream()),msg)
                local header = {http_response_header()}
                for i in pairs(header) do
                        write_msg("S-" .. tostring(tcp_stream()),"               " .. tostring(header[i]))
                end
        end
end

-- a listener tap's draw function is called every few seconds in the GUI
-- and at end of file (once) in tshark
function tap.draw()
        print("file processed, closing all dumpers")
        for file,dumper in pairs(dumpers) do
                dumper:flush()
                dumper:close()
        end
        dumpers = {}
end

-- a listener tap's reset function is called at the end of a live capture run,
-- when a file is opened, or closed.  Tshark never appears to call it.
function tap.reset()
end

Exercise 1

Merge the previous script with this one. Nothing stops you from doing 2 things at the same time!

Exercise 2

What can you do in Wireshark to help you decrypt other streams but 0?

Exercise 3

Can you add the whole http response to your files?