From 2e8f359be2cb2a0b2977d9694d2fca7d3da5c4a3 Mon Sep 17 00:00:00 2001 From: ajread4 Date: Thu, 15 Dec 2022 22:59:22 -0500 Subject: [PATCH 1/8] added some json functionality, can output to screen --- scripts/evtx_dump.py | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/evtx_dump.py b/scripts/evtx_dump.py index 2248f9d..eb2947b 100755 --- a/scripts/evtx_dump.py +++ b/scripts/evtx_dump.py @@ -19,6 +19,9 @@ # Version v0.1.1 import Evtx.Evtx as evtx import Evtx.Views as e_views +import os +import xmltodict +from typing import Dict def main(): @@ -28,14 +31,45 @@ def main(): description="Dump a binary EVTX file into XML.") parser.add_argument("evtx", type=str, help="Path to the Windows EVTX event log file") + parser.add_argument("-o","--output",type=str,help="Path to the output file") args = parser.parse_args() with evtx.Evtx(args.evtx) as log: - print(e_views.XML_HEADER) - print("") - for record in log.records(): - print(record.xml()) - print("") + + if (args.output): + if(os.path.splitext(args.output)[1]==".json"): + for record in log.records(): + data_dict=xmltodict.parse(record.xml()) #convert the xml to a dictionary + ''' + for event_system_key,event_system_value in data_dict['Event']['System'].items(): #loop through each key and value pair + if isinstance(data_dict['Event']['System'][str(event_system_key)],Dict): #if the dictionary is nested, enter the dictionary + sublist=[] + for event_system_subkey,event_system_subvalue in data_dict['Event']['System'][str(event_system_key)].items(): #loop through the nested dictionary + print(event_system_key+"_"+event_system_subkey[1:] + ":" + str(event_system_subvalue)) + else: + print(event_system_key + ":" + str(event_system_value)) + ''' + for event_system_key, event_system_value in data_dict['Event']['System'].items(): # loop through each key and value pair + if (event_system_key=="EventRecordID"): + json_subline = {} + firstline={event_system_key:event_system_value} + json_subline.update(firstline) + for event_data_key, event_data_value in data_dict['Event']['EventData'].items(): # loop through each key and value pair + for values in event_data_value: + for event_data_subkey,event_data_subvalue in values.items(): + if event_data_subkey=="@Name": + data_name=event_data_subvalue + else: + data_value=event_data_subvalue + json_subline.update({data_name:data_value}) + else: + print("Invalid File Type") + else: + print(e_views.XML_HEADER) + print("") + for record in log.records(): + print(record.xml()) + print("") if __name__ == "__main__": From df1f3fdced671ac8667b4d0e15434a41202d6b00 Mon Sep 17 00:00:00 2001 From: ajread4 Date: Sun, 18 Dec 2022 17:45:20 -0500 Subject: [PATCH 2/8] added JSON functionality and new dump file --- scripts/evtx_dump.py | 56 +++++++++++++++++++-------------------- scripts/evtx_dump_json.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 scripts/evtx_dump_json.py diff --git a/scripts/evtx_dump.py b/scripts/evtx_dump.py index eb2947b..5ab30b2 100755 --- a/scripts/evtx_dump.py +++ b/scripts/evtx_dump.py @@ -21,7 +21,7 @@ import Evtx.Views as e_views import os import xmltodict -from typing import Dict +import json def main(): @@ -31,39 +31,37 @@ def main(): description="Dump a binary EVTX file into XML.") parser.add_argument("evtx", type=str, help="Path to the Windows EVTX event log file") - parser.add_argument("-o","--output",type=str,help="Path to the output file") + parser.add_argument("-o","--output",type=str,help="Path to output JSON file") args = parser.parse_args() with evtx.Evtx(args.evtx) as log: if (args.output): - if(os.path.splitext(args.output)[1]==".json"): - for record in log.records(): - data_dict=xmltodict.parse(record.xml()) #convert the xml to a dictionary - ''' - for event_system_key,event_system_value in data_dict['Event']['System'].items(): #loop through each key and value pair - if isinstance(data_dict['Event']['System'][str(event_system_key)],Dict): #if the dictionary is nested, enter the dictionary - sublist=[] - for event_system_subkey,event_system_subvalue in data_dict['Event']['System'][str(event_system_key)].items(): #loop through the nested dictionary - print(event_system_key+"_"+event_system_subkey[1:] + ":" + str(event_system_subvalue)) - else: - print(event_system_key + ":" + str(event_system_value)) - ''' - for event_system_key, event_system_value in data_dict['Event']['System'].items(): # loop through each key and value pair - if (event_system_key=="EventRecordID"): - json_subline = {} - firstline={event_system_key:event_system_value} - json_subline.update(firstline) - for event_data_key, event_data_value in data_dict['Event']['EventData'].items(): # loop through each key and value pair - for values in event_data_value: - for event_data_subkey,event_data_subvalue in values.items(): - if event_data_subkey=="@Name": - data_name=event_data_subvalue - else: - data_value=event_data_subvalue - json_subline.update({data_name:data_value}) - else: - print("Invalid File Type") + final_json=[] + for record in log.records(): + data_dict=xmltodict.parse(record.xml()) #convert the xml to a dictionary + for event_system_key, event_system_value in data_dict['Event']['System'].items(): # loop through each key and value pair + if (event_system_key=="EventRecordID"): + json_subline={} + firstline={event_system_key:event_system_value} + json_subline.update(firstline) #add the event ID to JSON subline + for event_data_key, event_data_value in data_dict['Event']['EventData'].items(): # loop through each key and value pair + for values in event_data_value: + for event_data_subkey,event_data_subvalue in values.items(): #loop through each + if event_data_subkey=="@Name": #extract the name from the value + data_name=event_data_subvalue + else: + data_value=event_data_subvalue #extract the true value + json_subline.update({data_name:data_value}) #update the JSON sub line + final_json.append(json_subline) #update the final + + # Output the JSON data + if (os.path.splitext(args.output)[1] == ".json"): #if the file extension is correct + json_file=args.output + else: # if the file extension is incorrect + json_file=args.output +".json" + with open(json_file,"w") as outfile: #write to an output file + json.dump(final_json,outfile) else: print(e_views.XML_HEADER) print("") diff --git a/scripts/evtx_dump_json.py b/scripts/evtx_dump_json.py new file mode 100644 index 0000000..0145f2f --- /dev/null +++ b/scripts/evtx_dump_json.py @@ -0,0 +1,50 @@ +# Written by AJ Read with help from evtx_dump.py file. Adds functionality to dump EVTX to JSON. + +import Evtx.Evtx as evtx +import Evtx.Views as e_views +import os +import xmltodict +import json + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description="Dump a binary EVTX file into XML.") + parser.add_argument("evtx", type=str, + help="Path to the Windows EVTX event log file") + parser.add_argument("-o","--output",type=str,help="Path to output JSON file") + args = parser.parse_args() + + with evtx.Evtx(args.evtx) as log: + final_json=[] + for record in log.records(): + data_dict=xmltodict.parse(record.xml()) #convert the xml to a dictionary + for event_system_key, event_system_value in data_dict['Event']['System'].items(): # loop through each key and value pair + if (event_system_key=="EventRecordID"): + json_subline={} + firstline={event_system_key:event_system_value} + json_subline.update(firstline) #add the event ID to JSON subline + for event_data_key, event_data_value in data_dict['Event']['EventData'].items(): # loop through each key and value pair + for values in event_data_value: + for event_data_subkey,event_data_subvalue in values.items(): #loop through each + if event_data_subkey=="@Name": #extract the name from the value + data_name=event_data_subvalue + else: + data_value=event_data_subvalue #extract the true value + json_subline.update({data_name:data_value}) #update the JSON sub line + final_json.append(json_subline) #update the final + + # If output is desired + if (args.output): + # Output the JSON data + if (os.path.splitext(args.output)[1] == ".json"): #if the file extension is correct + json_file=args.output + else: # if the file extension is incorrect + json_file=args.output +".json" + with open(json_file,"w") as outfile: #write to an output file + json.dump(final_json,outfile) + else: + print(final_json) +if __name__ == "__main__": + main() From bba9e220c872f758631cef81a119ebc2241dfcb6 Mon Sep 17 00:00:00 2001 From: ajread4 Date: Sun, 18 Dec 2022 17:58:29 -0500 Subject: [PATCH 3/8] added comments and explanations to evtx_dump_json --- scripts/evtx_dump_json.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/evtx_dump_json.py b/scripts/evtx_dump_json.py index 0145f2f..f6cf42e 100644 --- a/scripts/evtx_dump_json.py +++ b/scripts/evtx_dump_json.py @@ -1,10 +1,12 @@ -# Written by AJ Read with help from evtx_dump.py file. Adds functionality to dump EVTX to JSON. +# Written by AJ Read with help from evtx_dump.py file +# Adds functionality to evtx_dump so that the user can dump evtx data formatted in JSON to the command line or a file. +# The JSON data uses only the "EventRecordID" from the "System" XML structure while using all the fields in the "EventData" xml structure. import Evtx.Evtx as evtx import Evtx.Views as e_views -import os -import xmltodict -import json +import os #added dependency +import xmltodict #added dependency +import json #added dependency def main(): import argparse From d833e402dab4ee7d0dbe329df2b630b3a333d084 Mon Sep 17 00:00:00 2001 From: ajread4 Date: Tue, 20 Dec 2022 14:50:36 -0500 Subject: [PATCH 4/8] added better comments, more testing, and reformatted output --- README.md | 2 +- scripts/evtx_dump.py | 43 +++-------------------- scripts/evtx_dump_json.py | 74 ++++++++++++++++++++++++++++----------- setup.py | 1 + 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index d984425..108f11d 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ python-evtx operates on event log files from Windows operating systems newer tha Examples -------- -Provided with the parsing module `Evtx` are three scripts that mimic the tools distributed with Parse-Evtx. `evtx_info.py` prints metadata about the event log and verifies the checksums of each chunk. `evtx_templates.py` builds and prints the templates used throughout the event log. Finally, `evtx_dump.py` parses the event log and transforms the binary XML into a human readable ASCII XML format. +Provided with the parsing module `Evtx` are four scripts that mimic the tools distributed with Parse-Evtx. `evtx_info.py` prints metadata about the event log and verifies the checksums of each chunk. `evtx_templates.py` builds and prints the templates used throughout the event log. `evtx_dump.py` parses the event log and transforms the binary XML into a human readable ASCII XML format. Finally, `evtx_dump_json.py` parses event logs, similar to `evtx_dump.py` and transforms the binary XML into JSON with the added capability to output new line delimited JSON to a file. Note the length of the `evtx_dump.py` script: its only 20 lines. Now, review the contents and notice the complete implementation of the logic: diff --git a/scripts/evtx_dump.py b/scripts/evtx_dump.py index 5ab30b2..8c975ef 100755 --- a/scripts/evtx_dump.py +++ b/scripts/evtx_dump.py @@ -19,10 +19,6 @@ # Version v0.1.1 import Evtx.Evtx as evtx import Evtx.Views as e_views -import os -import xmltodict -import json - def main(): import argparse @@ -31,43 +27,14 @@ def main(): description="Dump a binary EVTX file into XML.") parser.add_argument("evtx", type=str, help="Path to the Windows EVTX event log file") - parser.add_argument("-o","--output",type=str,help="Path to output JSON file") args = parser.parse_args() with evtx.Evtx(args.evtx) as log: - - if (args.output): - final_json=[] - for record in log.records(): - data_dict=xmltodict.parse(record.xml()) #convert the xml to a dictionary - for event_system_key, event_system_value in data_dict['Event']['System'].items(): # loop through each key and value pair - if (event_system_key=="EventRecordID"): - json_subline={} - firstline={event_system_key:event_system_value} - json_subline.update(firstline) #add the event ID to JSON subline - for event_data_key, event_data_value in data_dict['Event']['EventData'].items(): # loop through each key and value pair - for values in event_data_value: - for event_data_subkey,event_data_subvalue in values.items(): #loop through each - if event_data_subkey=="@Name": #extract the name from the value - data_name=event_data_subvalue - else: - data_value=event_data_subvalue #extract the true value - json_subline.update({data_name:data_value}) #update the JSON sub line - final_json.append(json_subline) #update the final - - # Output the JSON data - if (os.path.splitext(args.output)[1] == ".json"): #if the file extension is correct - json_file=args.output - else: # if the file extension is incorrect - json_file=args.output +".json" - with open(json_file,"w") as outfile: #write to an output file - json.dump(final_json,outfile) - else: - print(e_views.XML_HEADER) - print("") - for record in log.records(): - print(record.xml()) - print("") + print(e_views.XML_HEADER) + print("") + for record in log.records(): + print(record.xml()) + print("") if __name__ == "__main__": diff --git a/scripts/evtx_dump_json.py b/scripts/evtx_dump_json.py index f6cf42e..f3b1464 100644 --- a/scripts/evtx_dump_json.py +++ b/scripts/evtx_dump_json.py @@ -1,52 +1,84 @@ -# Written by AJ Read with help from evtx_dump.py file -# Adds functionality to evtx_dump so that the user can dump evtx data formatted in JSON to the command line or a file. -# The JSON data uses only the "EventRecordID" from the "System" XML structure while using all the fields in the "EventData" xml structure. +#!/usr/bin/env python3 +# This file is part of python-evtx. +# Written by AJ Read with help from evtx_dump.py file written by Willi Ballenthin. +# +# Purpose: User can dump evtx data into JSON format to either the command line or a JSON file in new line delimited format. import Evtx.Evtx as evtx import Evtx.Views as e_views -import os #added dependency -import xmltodict #added dependency -import json #added dependency + +# Added packages +import os +import xmltodict +import json + def main(): import argparse - parser = argparse.ArgumentParser( description="Dump a binary EVTX file into XML.") parser.add_argument("evtx", type=str, help="Path to the Windows EVTX event log file") - parser.add_argument("-o","--output",type=str,help="Path to output JSON file") + parser.add_argument("-o","--output",type=str, + help="Path of output JSON file") args = parser.parse_args() with evtx.Evtx(args.evtx) as log: + + # Instantiate the final json object final_json=[] + + # Loop through each record in the evtx log for record in log.records(): - data_dict=xmltodict.parse(record.xml()) #convert the xml to a dictionary - for event_system_key, event_system_value in data_dict['Event']['System'].items(): # loop through each key and value pair + + # Convert the record to a dictionary for ease of parsing + data_dict=xmltodict.parse(record.xml()) + + # Loop through each key,value pair of the System section of the evtx logs and extract the EventRecordID + for event_system_key, event_system_value in data_dict['Event']['System'].items(): if (event_system_key=="EventRecordID"): json_subline={} firstline={event_system_key:event_system_value} + + # Add information to the JSON object for this specific log json_subline.update(firstline) #add the event ID to JSON subline - for event_data_key, event_data_value in data_dict['Event']['EventData'].items(): # loop through each key and value pair + + # Loop through each key, value pair of the EventData section of the evtx logs + for event_data_key, event_data_value in data_dict['Event']['EventData'].items(): for values in event_data_value: - for event_data_subkey,event_data_subvalue in values.items(): #loop through each - if event_data_subkey=="@Name": #extract the name from the value + + # Loop through each subvalue within the EvenData section to extract necessary information + for event_data_subkey,event_data_subvalue in values.items(): + if event_data_subkey=="@Name": data_name=event_data_subvalue else: - data_value=event_data_subvalue #extract the true value - json_subline.update({data_name:data_value}) #update the JSON sub line - final_json.append(json_subline) #update the final + data_value=event_data_subvalue + + # Add information to the JSON object for this specific log + json_subline.update({data_name:data_value}) + + # Print the JSON object for the specific log if not requested to output to file + if not args.output: + print(json_subline) + + # Add specific log JSON object to the final JSON object + if not final_json: + final_json=[json_subline] + else: + final_json.append(json_subline) # If output is desired if (args.output): + # Output the JSON data - if (os.path.splitext(args.output)[1] == ".json"): #if the file extension is correct + if (os.path.splitext(args.output)[1] == ".json"): json_file=args.output - else: # if the file extension is incorrect + else: json_file=args.output +".json" - with open(json_file,"w") as outfile: #write to an output file + + # Write to JSON file + with open(json_file,"w") as outfile: json.dump(final_json,outfile) - else: - print(final_json) + if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index 034a96a..776ab10 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ ] }, scripts=['scripts/evtx_dump.py', + 'scripts/evtx_dump_json.py' 'scripts/evtx_dump_chunk_slack.py', 'scripts/evtx_eid_record_numbers.py', 'scripts/evtx_extract_record.py', From 7ac40e8a6d0d6cfe05232f6be2f0a129e50b9f27 Mon Sep 17 00:00:00 2001 From: ajread4 Date: Tue, 20 Dec 2022 14:57:02 -0500 Subject: [PATCH 5/8] fixed help menu and tested default values --- scripts/evtx_dump_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/evtx_dump_json.py b/scripts/evtx_dump_json.py index f3b1464..3e8977a 100644 --- a/scripts/evtx_dump_json.py +++ b/scripts/evtx_dump_json.py @@ -7,7 +7,7 @@ import Evtx.Evtx as evtx import Evtx.Views as e_views -# Added packages +# Added packages import os import xmltodict import json @@ -17,9 +17,9 @@ def main(): import argparse parser = argparse.ArgumentParser( description="Dump a binary EVTX file into XML.") - parser.add_argument("evtx", type=str, + parser.add_argument("evtx", type=str,action='store', help="Path to the Windows EVTX event log file") - parser.add_argument("-o","--output",type=str, + parser.add_argument("-o","--output",type=str, action='store', help="Path of output JSON file") args = parser.parse_args() From df24946f172566797dd1f4a0ca94efff7fffd3ff Mon Sep 17 00:00:00 2001 From: ajread4 Date: Tue, 20 Dec 2022 15:07:34 -0500 Subject: [PATCH 6/8] finished with updates to dump_json --- README.md | 2 +- scripts/evtx_dump_json.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 108f11d..9f35257 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ python-evtx operates on event log files from Windows operating systems newer tha Examples -------- -Provided with the parsing module `Evtx` are four scripts that mimic the tools distributed with Parse-Evtx. `evtx_info.py` prints metadata about the event log and verifies the checksums of each chunk. `evtx_templates.py` builds and prints the templates used throughout the event log. `evtx_dump.py` parses the event log and transforms the binary XML into a human readable ASCII XML format. Finally, `evtx_dump_json.py` parses event logs, similar to `evtx_dump.py` and transforms the binary XML into JSON with the added capability to output new line delimited JSON to a file. +Provided with the parsing module `Evtx` are four scripts that mimic the tools distributed with Parse-Evtx. `evtx_info.py` prints metadata about the event log and verifies the checksums of each chunk. `evtx_templates.py` builds and prints the templates used throughout the event log. `evtx_dump.py` parses the event log and transforms the binary XML into a human readable ASCII XML format. Finally, `evtx_dump_json.py` parses event logs, similar to `evtx_dump.py` and transforms the binary XML into JSON with the added capability to output the JSON array to a file. Note the length of the `evtx_dump.py` script: its only 20 lines. Now, review the contents and notice the complete implementation of the logic: diff --git a/scripts/evtx_dump_json.py b/scripts/evtx_dump_json.py index 3e8977a..63888ac 100644 --- a/scripts/evtx_dump_json.py +++ b/scripts/evtx_dump_json.py @@ -2,7 +2,8 @@ # This file is part of python-evtx. # Written by AJ Read with help from evtx_dump.py file written by Willi Ballenthin. # -# Purpose: User can dump evtx data into JSON format to either the command line or a JSON file in new line delimited format. +# Purpose: User can dump evtx data into JSON format to either the command line or a JSON file in new line delimited format/JSON array. +# Details: The JSON object is created with only the EventRecordID from the System section of the evtx XML and all of the information within the EventData section. import Evtx.Evtx as evtx import Evtx.Views as e_views From 49d5117b7656943024e5a28900ad7657029bff65 Mon Sep 17 00:00:00 2001 From: ajread4 Date: Tue, 20 Dec 2022 17:15:32 -0500 Subject: [PATCH 7/8] fixed the issue with corrupted first line in EventData section --- scripts/evtx_dump_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/evtx_dump_json.py b/scripts/evtx_dump_json.py index 63888ac..a1f7744 100644 --- a/scripts/evtx_dump_json.py +++ b/scripts/evtx_dump_json.py @@ -55,9 +55,9 @@ def main(): else: data_value=event_data_subvalue - # Add information to the JSON object for this specific log - json_subline.update({data_name:data_value}) - + # Add information to the JSON object for this specific log + json_subline.update({data_name: data_value}) + # Print the JSON object for the specific log if not requested to output to file if not args.output: print(json_subline) From b0204d1f9e75f98a72adc56b49ccb43d4e6d2fb0 Mon Sep 17 00:00:00 2001 From: ajread4 Date: Wed, 21 Dec 2022 08:53:59 -0500 Subject: [PATCH 8/8] added dependency in setup.py --- scripts/evtx_dump_json.py | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/evtx_dump_json.py b/scripts/evtx_dump_json.py index a1f7744..68d1c2d 100644 --- a/scripts/evtx_dump_json.py +++ b/scripts/evtx_dump_json.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # This file is part of python-evtx. -# Written by AJ Read with help from evtx_dump.py file written by Willi Ballenthin. +# Written by AJ Read (ajread4) with help/inspiration from the evtx_dump.py file written by Willi Ballenthin. # # Purpose: User can dump evtx data into JSON format to either the command line or a JSON file in new line delimited format/JSON array. # Details: The JSON object is created with only the EventRecordID from the System section of the evtx XML and all of the information within the EventData section. @@ -57,7 +57,7 @@ def main(): # Add information to the JSON object for this specific log json_subline.update({data_name: data_value}) - + # Print the JSON object for the specific log if not requested to output to file if not args.output: print(json_subline) diff --git a/setup.py b/setup.py index 776ab10..36a3dbf 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ install_requires=[ 'six', 'hexdump==3.3', + 'xmltodict==0.13.0', #added deps for evtx_dump_json.py script # pin deps for python 2, see #67 'more_itertools==5.0.0',