Reading a log file to add information to your streams¶
We are now going a open an APM log file, extract session ID value and usernames.
Script:
-- Let's register all the fields we want TShark to extract
-- Remember that we are creating functions retrieving the field values.
local tcp_stream = Field.new("tcp.stream")
local http_method = Field.new("http.request.method")
local http_uri = Field.new("http.request.uri")
local http_version = Field.new("http.request.version")
local http_code = Field.new("http.response.code")
local http_phrase = Field.new("http.response.phrase")
local http_location = Field.new("http.location")
local http_request = Field.new("http.request")
local http_response = Field.new("http.response")
local http_cookie = Field.new("http.cookie_pair")
local http_setcookie = Field.new("http.set_cookie")
local http_request_header = Field.new("http.request.line")
local http_response_header = Field.new("http.response.line")
local http_response_data = Field.new("http.file_data")
-- Creating the listener, to avoid http data appearing twice due to retransmit, let's suppress them
local tap = Listener.new(nil,"http && !tcp.analysis.retransmission && !tcp.analysis.lost_segment")
local dumpers = {}
-- To avoid runtime error due to nil variable, this function will simply display nothing if TShark has not found the field we are looking for.
local function to_string(string)
if (string == nil) then
return ""
else
return tostring(string)
end
end
-- Same as the previous example, but this time we are writing text with normal LUA I/O functions.
local function write_msg(id,msg)
local file = dumpers[id]
if not file then
file = io.open("streams/" .. id .. ".txt", "a")
if (file == nil) then
for file,dumper in pairs(dumpers) do
dumper:flush()
dumper:close()
end
dumpers = {}
file = assert(io.open("streams/" .. id .. ".txt", "a"))
end
dumpers[id] = file
end
file:write(msg)
end
-- Let's create some arrays that will be useful when reading our APM log files.
local timeouts = {}
local months = {}
local maps = {}
-- Need to convert months as found in the log file to something LUA will understand
months["Jan"] = 1
months["Feb"] = 2
months["Mar"] = 3
months["Apr"] = 4
months["May"] = 5
months["Jun"] = 6
months["Jul"] = 7
months["Aug"] = 8
months["Sep"] = 9
months["Oct"] = 10
months["Nov"] = 11
months["Dec"] = 12
local apm = assert(io.open("apm", "r"))
for l in apm:lines() do
-- Jun 3 19:31:15 pulsar notice tmm1[10766]: 01490520:5: /Common/test:Common:216342fe: Session deleted due to admin initiated termination.
local month, day, h, m, s, session, reason = l:match '(%w+) (%d+) (%d+):(%d+):(%d+).*Common:(%w+): Session deleted due to (%w+) .*'
if (session ~= nil) then
-- There is no year to extract from the log file
local convertTime = os.time({year = "2017", month = months[month], day = day, hour = h, min = m, sec = s})
-- You may need to offset the timestamps
-- convertTime = convertTime - 3600
-- Adding the expiry timestamp (and reason) for a given session to our table
timeouts[session] = {convertTime, reason}
end
-- Jun 3 19:30:13 pulsar notice apmd[6566]: 01490010:5: /Common/test:Common:ab9bee05: Username 'u-0088979'
local session, username = l:match '.*Common:(%w+): Username \'(.*)\''
if (username ~= nil) then
-- Adding the our username for a given session to our table, sometimes sessions are created with no Username, so lets ignore those.
maps[session] = username
end
end
print("Done processing APM log file")
function tap.packet(pinfo,tvb,tapdata)
-- The only reason I am checking if this is a http request is due to the cookies
if ( http_request() ) then
if( http_method() == nil ) then return end
local request_mrhsession = ""
local cookie = {http_cookie()}
for i in pairs(cookie) do
-- One of the powerful function of LUA is its text matching
request_mrhsession = tostring(cookie[i]):match '.*LastMRH_Session=(%w+).*'
if (request_mrhsession ~= nil ) then break end
end
-- What we are doing here is to compare the current timestamp of our packet with an APM session
local username = ""
if ( request_mrhsession ~= nil and maps[request_mrhsession] ~= nil ) then
-- retrieve the username from the LastMRH_Session cookie
username = maps[request_mrhsession]
-- do we know when this session was expired
if(timeouts[request_mrhsession] ~= nil) then
-- If the packet timestamps is superior, then the session has been previously deleted, and that user created a new session
if(tonumber(pinfo.abs_ts) > tonumber(timeouts[request_mrhsession][1])) then
local msg = "** " .. string.format("%8d", pinfo.number) .. " ** " .. os.date("%b %d, %Y %X",timeouts[request_mrhsession][1]) .. ".000000000 PDT +++++++ THIS USER HAS HAD HIS SESSION TERMINATED BY " .. timeouts[request_mrhsession][2] .. "\n"
write_msg(username,msg)
-- Lets remove that key from our map table, otherwise we will keep writing the message above all the time
timeouts[request_mrhsession] = nil
end
end
end
local msg =
to_string("** " .. string.format("%8d", pinfo.number)) .. " ** " ..
to_string(format_date(pinfo.abs_ts)) .. " " ..
to_string(http_method()) .. " " ..
to_string(http_uri()) .. " " ..
to_string(http_version()) .. " " ..
to_string(request_mrhsession) .. " " ..
to_string(username) .. " " ..
to_string(tcp_stream())
.. "\n"
-- Writing to file the request for a given username, since we know it. That been said, there is a limit here an initial request with no MRH will not match.
-- We would need to wait for the response and then retrieve the corresponding request to finally write it to file.
-- RFE: Need to store the request in memory with the stream ID, and probably a request number. Wait for the matching response and commit to file.
if ( username ~= "" ) then
write_msg(username,msg)
else
write_msg("u-UNKNOWN",msg)
end
-- Writing to file all of our HTTP request headers, we are adding a "S-" prefix to the filename
write_msg("S-" .. tostring(tcp_stream()),msg)
local header = {http_request_header()}
for i in pairs(header) do
write_msg("S-" .. tostring(tcp_stream())," " .. tostring(header[i]))
end
else
-- If that is not a request, then it is a response :)
local response_mrhsession = ""
local cookie = {http_setcookie()}
for i in pairs(cookie) do
response_mrhsession = tostring(cookie[i]):match '.*LastMRH_Session=(%w+).*'
if (response_mrhsession ~= nil ) then break end
end
local username = ""
if ( response_mrhsession ~= nil and maps[response_mrhsession] ~= nil ) then
username = maps[response_mrhsession]
end
local msg =
to_string("** " .. string.format("%8d", pinfo.number)) .. " ** " ..
to_string(format_date(pinfo.abs_ts)) .. " " ..
to_string(http_code()) .. " " ..
to_string(http_phrase()) .. " " ..
to_string(http_version()) .. " " ..
to_string(response_mrhsession) .. " " ..
to_string(username) .. " " ..
to_string(http_location()) .. " " ..
to_string(tcp_stream())
.. "\n"
if ( username ~= "" ) then
write_msg(username,msg)
else
write_msg("u-UNKNOWN",msg)
end
write_msg("S-" .. tostring(tcp_stream()),msg)
local header = {http_response_header()}
for i in pairs(header) do
write_msg("S-" .. tostring(tcp_stream())," " .. tostring(header[i]))
end
-- write_msg("S-" .. tostring(tcp_stream()),to_string(http_response_data()))
end
end
-- a listener tap's draw function is called every few seconds in the GUI
-- and at end of file (once) in tshark
function tap.draw()
print("file processed, closing all dumpers")
for file,dumper in pairs(dumpers) do
dumper:flush()
dumper:close()
end
dumpers = {}
print("adding last user timeouts to log files")
for key,value in pairs(timeouts) do
local username = maps[key]
if (username ~= nil) then
local msg = "** - ** " .. os.date("%b %d, %Y %X",value[1]) .. ".000000000 PDT ####### THIS USER HAS HAD HIS SESSION TERMINATED BY " .. value[2] .. "\n"
local file = io.open("streams/" .. username .. ".txt", "r")
if (file ~= nil) then
file:close()
local file = io.open("streams/" .. username .. ".txt", "a")
file:write(msg)
file:close()
end
end
end
end
-- a listener tap's reset function is called at the end of a live capture run,
-- when a file is opened, or closed. Tshark never appears to call it.
function tap.reset()
end
Exercise 1¶
We are not adding the timeouts to our files, using your S-0.cap file try to workout why.