Draft: Start implementing writer

.gitignore

@@ -2,4 +2,5 @@
 .vscode/
 protoc
 protoc-gen-go
-protoc-go-inject-tag
+protoc-go-inject-tag
+data/*.pst

README.md

@@ -6,7 +6,7 @@
   <br>
 </h1>
 
-<h4 align="center">A library for reading PST files (written in Go/Golang)</h4>
+<h4 align="center">A library for reading and writing PST files (written in Go/Golang)</h4>
 
 <p align="center">
   <a href="https://github.com/mooijtech/go-pst/blob/master/LICENSE.txt">
@@ -30,12 +30,16 @@
 
 The PFF (Personal Folder File) and OFF (Offline Folder File) format is used to store Microsoft Outlook e-mails, appointments and contacts. The PST (Personal Storage Table), OST (Offline Storage Table) and PAB (Personal Address Book) file format consist of the PFF format.
 
+Created for [Go Forensics](https://goforensics.io/) by [Marten Mooij](#contact).
+
 ## Usage
 
 ```bash
 $ go install github.com/mooijtech/go-pst/v6
 ```
 
+### Reader
+
 ```go
 package main
 
@@ -173,9 +177,15 @@ func main() {
 }
 ```
 
+### Writer
+
+```go
+
+```
+
 ## License 
 
-This project is licensed under the [Apache License 2.0]().
+This project is licensed under the [Apache License 2.0](https://github.com/mooijtech/go-pst/blob/master/LICENSE.txt).
 
 ## Documentation
 
@@ -184,10 +194,14 @@ This project is licensed under the [Apache License 2.0]().
 
 ## Implementations
 
-- [java-libpst](https://github.com/rjohnsondev/java-libpst)
-- [pstreader](https://github.com/Jmcleodfoss/pstreader)
-  - Special thanks to [James McLeod](https://github.com/Jmcleodfoss)
-- [libpff](https://github.com/libyal/libpff)
+- [java-libpst](https://github.com/rjohnsondev/java-libpst) (Richard Johnson)
+  - Inspired me to write go-pst
+- [pstreader](https://github.com/Jmcleodfoss/pstreader) (James McLeod)
+  - Kindly replied to my questions
+  - Inspired go-pst property generation
+- [libpff](https://github.com/libyal/libpff) (Joachim Metz)
+  - Original Gangster
+  - Inspired java-libpst
 - [XstReader](https://github.com/Dijji/XstReader)
 - [PANhunt](https://github.com/Dionach/PANhunt/blob/master/pst.py)
 

cmd/example-attachment.txt

@@ -0,0 +1,15 @@
+// go-pst is a library for reading Personal Storage Table (.pst) files (written in Go/Golang).
+//
+// Copyright 2023 Marten Mooij
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.

cmd/properties/folder.proto

@@ -0,0 +1,25 @@
+// go-pst is a library for reading Personal Storage Table (.pst) files (written in Go/Golang).
+//
+// Copyright 2023 Marten Mooij
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:generate msgp -tests=false
+
+syntax = "proto3";
+option go_package = "github.com/mooijtech/go-pst;properties";
+
+message Folder {
+  string name = 1;
+  string has_subfolders = 2;
+}

cmd/properties/generate.go

@@ -1,19 +1,18 @@
 // go-pst is a library for reading Personal Storage Table (.pst) files (written in Go/Golang).
 //
-// Copyright (C) 2022  Marten Mooij
+// Copyright 2023 Marten Mooij
 //
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// http://www.apache.org/licenses/LICENSE-2.0
 //
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package main
 
@@ -38,48 +37,52 @@ import (
 
 // main starts the Protocol Buffers generation.
 func main() {
+	// Download [MS-OXPROPS].
 	downloadedFile, err := download()
 
 	if err != nil {
-		panic(fmt.Sprintf("Failed to download [MS-OXPROPS].docx: %+v", errors.WithStack(err)))
+		panic(fmt.Sprintf("Failed to download [MS-OXPROPS].docx: %+v", err))
 	}
 
 	defer func() {
 		if err = os.Remove(downloadedFile); err != nil {
-			panic(fmt.Sprintf("Failed to remove [MS-OXPROPS].docx: %+v", errors.WithStack(err)))
+			panic(fmt.Sprintf("Failed to remove [MS-OXPROPS].docx: %+v", err))
 		}
 	}()
 
 	// TODO - Verify hash.
 
+	// Unzip as .DOCX is actually just zipped XML.
 	unzippedFile, err := unzip(downloadedFile)
 
 	if err != nil {
-		panic(fmt.Sprintf("Failed to unzip [MS-OXPROPS].docx: %+v", errors.WithStack(err)))
+		panic(fmt.Sprintf("Failed to unzip [MS-OXPROPS].docx: %+v", err))
 	}
 
 	defer func() {
 		if err = os.Remove(unzippedFile); err != nil {
-			panic(fmt.Sprintf("Failed to remove unzipped file [MS-OXPROPS].xml: %+v", errors.WithStack(err)))
+			panic(fmt.Sprintf("Failed to remove unzipped file [MS-OXPROPS].xml: %+v", err))
 		}
 	}()
 
+	// Get all defined properties.
 	properties, err := getPropertiesFromXML(unzippedFile)
 
 	if err != nil {
 		panic(fmt.Sprintf("Failed to create properties: %+v", errors.WithStack(err)))
 	}
 
+	// Create .proto files from the defined properties.
 	if err = generateProtocolBuffers(properties); err != nil {
 		panic(fmt.Sprintf("Failed to generate Protocol Buffers: %+v", errors.WithStack(err)))
 	}
 
-	// TODO - Check if exists.
+	// Create a property map used by go-pst to map values from the Property Context to their Property Type.
 	// Write property map.
 	propertyMap, err := os.Create("properties.csv")
 
 	if err != nil {
-		panic(errors.WithStack(err))
+		panic(fmt.Sprintf("Failed to create output file: %+v", err))
 	}
 
 	propertyMapWriter := csv.NewWriter(propertyMap)
@@ -104,10 +107,19 @@ func main() {
 	propertyMapWriter.Flush() // Important since writes are buffered.
 
 	if propertyMapWriter.Error() != nil {
-		panic(errors.WithStack(propertyMapWriter.Error()))
+		panic(fmt.Sprintf("Failed to write property map: %+v", propertyMapWriter.Error()))
 	} else if err := propertyMap.Close(); err != nil {
-		panic(errors.WithStack(err))
+		panic(fmt.Sprintf("Failed to close property map: %+v", err))
 	}
+
+	// TODO - Validate properties exist, parse DOCX in the same way:
+	// TODO - [MS-OXCMSG]: Message and Attachment Object Protocol
+	// TODO - [MS-OXCFOLD]: Folder Object Protocol and
+	// TODO - See referenceToProtocolBufferMessageType for mappings.
+	// TODO - Extend properties.Folder to include everything in [MS-OXCFOLD]: Folder Object Protocol.
+	//for downloadName, downloadURL := referenceToProtocolBufferMessageType {
+	// TODO - Download and apply same parser logic.
+	//}
 }
 
 // download the latest version of [MS-OXPROPS].docx and returns the downloaded file.
@@ -118,7 +130,7 @@ func download() (destinationFilename string, err error) {
 		Timeout: 60 * time.Second,
 	}
 
-	response, err := httpClientWithTimeout.Get("https://interoperability.blob.core.windows.net/files/MS-OXPROPS/%5bMS-OXPROPS%5d-210817.docx")
+	response, err := httpClientWithTimeout.Get("https://msopenspecs.azureedge.net/files/MS-OXPROPS/%5bMS-OXPROPS%5d-210817.docx")
 
 	if err != nil {
 		return "", errors.WithStack(err)
@@ -455,42 +467,106 @@ var propertyTypeToProtocolBufferFieldType = map[uint16]string{
 // areaNameToProtocolBufferMessageType defines the mapping from area names to Protocol Buffer Message Types.
 // TODO - Layer on top of "Common" area name, check the defining reference then sort accordingly.
 var areaNameToProtocolBufferMessageType = map[string]string{
-	"Sharing":                       "Sharing",
-	"Extracted Entities":            "Extracted Entity",
-	"Unified Messaging":             "Voicemail",
-	"Email":                         "Message",
-	"Journal":                       "Journal",
-	"Tasks":                         "Task",
-	"RSS":                           "RSS",
-	"Meetings":                      "Appointment",
-	"Calendar":                      "Appointment",
-	"Conferencing":                  "Appointment",
-	"Spam":                          "Spam",
-	"Reminders":                     "Appointment",
-	"General Message Properties":    "Message",
-	"Message Properties":            "Message",
-	"Message Time Properties":       "Message",
-	"Message Attachment Properties": "Attachment",
-	"Sticky Notes":                  "Note",
-	"Secure Messaging Properties":   "Message", // [MS-OXORMMS]: Rights-Managed Email Object Protocol.
-	"SMS":                           "SMS",     // [MS-OXOSMMS]: Short Message Service (SMS) and Multimedia Messaging Service (MMS) Object Protocol.
-	"Contact Properties":            "Contact",
-	"MapiMailUser":                  "Contact",
-	"Address Book":                  "Address Book", // [MS-OXOABK]: Address Book Object Protocol.
-	"MapiAddressBook":               "Address Book",
-	"Free/Busy Properties":          "Appointment",
-	"Archive":                       "Message",
-	"MapiRecipient":                 "Message",
-	"MIME Properties":               "Message",
-	"Address Properties":            "Message",
+	"Sharing":                             "Sharing",
+	"Extracted Entities":                  "Extracted Entity",
+	"Unified Messaging":                   "Voicemail",
+	"Email":                               "Message",
+	"Journal":                             "Journal",
+	"Tasks":                               "Task",
+	"RSS":                                 "RSS",
+	"Meetings":                            "Appointment",
+	"Calendar":                            "Appointment",
+	"Conferencing":                        "Appointment",
+	"Spam":                                "Spam",
+	"Reminders":                           "Appointment",
+	"General Message Properties":          "Message",
+	"Message Properties":                  "Message",
+	"Message Time Properties":             "Message",
+	"Message Attachment Properties":       "Attachment",
+	"Sticky Notes":                        "Note",
+	"Secure Messaging Properties":         "Message", // [MS-OXORMMS]: Rights-Managed Email Object Protocol.
+	"SMS":                                 "SMS",     // [MS-OXOSMMS]: Short Message Service (SMS) and Multimedia Messaging Service (MMS) Object Protocol.
+	"Contact Properties":                  "Contact",
+	"MapiMailUser":                        "Contact",
+	"Address Book":                        "Address Book", // [MS-OXOABK]: Address Book Object Protocol.
+	"MapiAddressBook":                     "Address Book",
+	"Free/Busy Properties":                "Appointment",
+	"Archive":                             "Message",
+	"MapiRecipient":                       "Message",
+	"MIME Properties":                     "Message",
+	"Address Properties":                  "Message",
+	"Common":                              "Message",
+	"Access Control Properties":           "Message",
+	"Outlook Application":                 "Message",
+	"Address book":                        "Address Book",
+	"History Properties":                  "Message",
+	"Sync":                                "Message",
+	"ICS":                                 "Message",
+	"Folder Properties":                   "Folder",
+	"Conversations":                       "Message",
+	"Configuration":                       "Message",
+	"Miscellaneous Properties":            "Message",
+	"MapiNonTransmittable":                "Message",
+	"Logon Properties":                    "Message",
+	"Mail":                                "Message",
+	"MessageClassDefinedTransmittable":    "Message",
+	"MapiMessageStore":                    "Message", // TODO - Message Store?
+	"ExchangeMessageStore":                "Message", // TODO - Message Store?
+	"Message Store Properties":            "Message",
+	"Appointment":                         "Appointment",
+	"MapiEnvelope":                        "Message",
+	"ExchangeNonTransmittableReserved":    "Message",
+	"Exchange Administrative":             "Message",
+	"TransportEnvelope":                   "Message",
+	"MapiNonTransmittable Property set":   "Message",
+	"MapiContainer":                       "Message",
+	"MapiAttachment":                      "Attachment",
+	"Rules":                               "Message",
+	"MapiStatus":                          "Message",
+	"Server-Side Rules Properties":        "Message",
+	"ID Properties":                       "Message",
+	"AB Container":                        "Message",
+	"Search":                              "Message",
+	"MapiEnvelope Property set":           "Message",
+	"Transport Envelope":                  "Message",
+	"Common Property set":                 "Message",
+	"Conflict Note":                       "Message",
+	"Table Properties":                    "Message",
+	"MAPI Display Tables":                 "Message",
+	"Server":                              "Message",
+	"General Report Properties":           "Message",
+	"Run-time configuration":              "Message",
+	"Meeting Response":                    "Meeting", // TODO - Meeting?
+	"Calendar Property set":               "Message", // TODO - Calendar?
+	"Site Mailbox":                        "Message",
+	"Flagging":                            "Message",
+	"Offline Address Book Properties":     "Address Book",
+	"MIME properties":                     "Message",
+	"BestBody":                            "Message",
+	"TransportRecipient":                  "Message",
+	"ExchangeAdministrative":              "Message",
+	"ProviderDefinedNonTransmittable":     "Message",
+	"MapiMessage":                         "Message",
+	"MapiCommon":                          "Message",
+	"ExchangeFolder":                      "Folder",
+	"ExchangeMessageReadOnly":             "Message",
+	"Message Class Defined Transmittable": "Message",
 }
 
 // referenceToProtocolBufferMessageType maps the defining reference prefix to the message type.
 // Preferred over area name mapping.
+// TODO - Use these documents directly :)
 var referenceToProtocolBufferMessageType = map[string]string{
-	"[MS-OXOMSG]":  "Message",
+	// References https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxomsg/daa9120f-f325-4afb-a738-28f91049ab3c
+	"[MS-OXOMSG]": "Message",
+	// References https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxocntc/9b636532-9150-4836-9635-9c9b756c9ccf
 	"[MS-OXOCNTC]": "Contact",
-	"[MS-OXOCAL]":  "Appointment",
+	// References https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxocal/09861fde-c8e4-4028-9346-e7c214cfdba1
+	"[MS-OXOCAL]": "Appointment",
+	// References https://learn.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxcfold/c0f31b95-c07f-486c-98d9-535ed9705fbf
+	"[MS-OXCFOLD]": "Folder",
+	// References
+	"[MS-OXOABK]": "Address Book Object Protocol",
 }
 
 //go:embed header.txt
@@ -513,6 +589,7 @@ func generateProtocolBuffers(properties []xmlProperty) error {
 		if protocolBufferMessageType == "" {
 			// We need to map this.
 			fmt.Printf("Unmapped area name \"%s\", defining reference: %s\n", property.AreaName, property.DefiningReference)
+			fmt.Println("Writing to unknown...")
 		} else {
 			protocolBufferBuilder := protocolBufferMessageTypeToBuilder[protocolBufferMessageType]
 
@@ -561,7 +638,7 @@ func createProtoFileField(property xmlProperty, protocolBufferFieldType string,
 
 		if property.PropertyID != 0 {
 			// TODO - There may be multiple property types.
-			goTags.WriteString(fmt.Sprintf("// @gotags: msg:\"%d%d,omitempty\"", property.PropertyID, property.PropertyTypes[0]))
+			goTags.WriteString(fmt.Sprintf("// @gotags: msg:\"%d-%d,omitempty\"", property.PropertyID, property.PropertyTypes[0]))
 		} // PropertySets are mapped via the PropertyMap in property_context.go
 
 		// Write.

cmd/properties/go.mod

@@ -2,4 +2,7 @@ module github.com/mooijtech/go-pst/generate/properties
 
 go 1.19
 
-require github.com/pkg/errors v0.9.1
+require (
+	github.com/pkg/errors v0.9.1
+	github.com/rotisserie/eris v0.5.4
+)