Merge pull request #1 from tycho01/master

test pull

CrossUsage.m

@@ -0,0 +1,11 @@
+let
+    Table.CrossJoin = Load("Table.Crossjoin"),
+    Tokens = Table.RenameColumns(Table.FromList(Record.FieldNames(#shared)), {"Column1", "Token"}),
+    AddAlts = Table.AddColumn(Tokens, "TokenAlt", each Text.Replace([Token], "_", ".")),
+    Crossed = Table.CrossJoin(AddAlts, UdfContents),
+    Contained = Table.AddColumn(Crossed, "Contains", each Text.Contains([Contents], [Token]) or Text.Contains([Contents], [TokenAlt])),
+    Filtered = Table.SelectRows(Contained, each [Contains]),
+    FiltCols = Table.SelectColumns(Filtered, {"Token", "TokenAlt", "Name"}),
+    Return = FiltCols
+in
+Return

F.m

@@ -0,0 +1,53 @@
+/*
+F (short for function), like Load(), provides one calling interface to access functions either imported (faster) or loaded (fallback), so code could be left agnostic to whether the queries are available locally, though whether this is really necessary is left up to the user to decide. One reasonable use case would be allowing many workbooks to access a shared library of queries from their files without needing to import them to each workbook after every change.
+
+Imported or loadable functions could be referenced as any of the following:
+	Load("Text.ReplaceAll")
+	Load("Text_ReplaceAll")
+	F[Text.ReplaceAll]
+	F[Text_ReplaceAll]
+
+Intended benefits of F over Load():
+	- shorter
+	- may help avoiding duplicate executions of file imports (if applicable) -- needs further testing though.
+*/
+
+let
+    Shared = #shared,
+
+
+    //Record.Rename = Load("Record.Rename"),
+    //Record.Rename = Record_Rename,
+/*
+    Record.Rename = (Rec as record, Lambda as function) as record =>
+    let
+        Keys = Record.FieldNames(Rec),
+        Values = Record.FieldValues(Rec),
+        Renamed = List.Transform(Keys, each Lambda(_, Record.Field(Rec, _))),
+        Recorded = Record.FromList(Values, Renamed)
+    in
+        Recorded,
+*/
+    //cyclic reference...?
+    //SharedPeriods = Record.Rename(Shared, (k,v) => Text.Replace(k, "_", ".")),
+
+    SharedPeriods = Record.FromList(Record.FieldValues(Shared), List.Transform(Record.FieldNames(Shared), each Text.Replace(_, "_", "."))),
+
+    SharedMerged = Record.Combine({Shared, SharedPeriods}),
+    //If I can make a wrapper function to enable profiling/persistence, wrap these as well
+
+    Files = Folder.Files(LoadPath),
+    MFiles = Table.SelectRows(Files, each Text.Lower([Extension]) = ".m"
+        // and [Folder Path] = LoadPath    // non-recursive
+    ),
+    NoExt = Table.TransformColumns(MFiles, {"Name", each Text.Start(_, Text.Length(_)-2)}),
+    CustomNames = Table.Column(NoExt, "Name"),
+    Underscored = List.Transform(CustomNames, each Text.Replace(_, ".", "_")),
+    CustomBoth = List.Union({CustomNames, Underscored}),
+    CustomLoaded = Record.FromList(List.Transform(CustomBoth, Load), CustomBoth),
+    SharedAndLoaded = Record.Combine({CustomLoaded, SharedMerged}),
+
+    Return = SharedAndLoaded
+in
+    Return
+

Files.SheetsInFolder.m

@@ -5,7 +5,8 @@
     Source = Files.SheetsInFolder("C:\path\to\my\folder\")
 //Result: [a table containing the binary file content, file names, sheet tables, and sheet names for each sheet in each spreadsheet in the given folder]
 */
-let SheetsInFolder = (folderPath as text) =>
+
+(folderPath as text) as table =>
 let
     Source = Folder.Files(folderPath),
     FilteredRows = Table.SelectRows(Source, each Text.Start([Extension],3) = ".xl"),
@@ -16,4 +17,3 @@
     NoPrintAreas = Table.SelectRows(Expanded, each not Text.Contains([N], "$"))
 in
     NoPrintAreas
-in SheetsInFolder

Function.Profile.m

@@ -0,0 +1,18 @@
+/*
+// Profiles the time taken to execute a function for the given parameters
+//Usage:
+    Text.Between = Load("Text.Between"),
+    Function.Profile = Load("Function.Profile"),
+    Function.Profile(Text.Between, {"abcdef", "bc", "f"})
+//Result: "de" meta 00:00:00
+*/
+
+(fn as function, params as list) as datetime =>
+let
+    TimeBefore = DateTime.LocalNow(),
+	evaluated = Function.Invoke(fn, params),
+    TimeAfter = (try evaluated as none otherwise DateTime.LocalNow()),
+// ^ always evaluates to otherwise, just using the expression as a dummy to force getting the time only after evaluation has finished
+    TimeTaken = TimeAfter - TimeBefore
+in
+    evaluated meta [taken=TimeTaken]

LICENSE

@@ -1,21 +1,21 @@
-The MIT License (MIT)
-
-Copyright (c) 2014 tycho01
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+The MIT License (MIT)
+
+Copyright (c) 2014 tycho01
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

List.CrossJoin.m

@@ -1,22 +1,20 @@
-/*
-//Returns the Cartesian product (i.e. crossjoin) of two arguments (can be lists or tables).
-//Usage:
-    List.CrossJoin = Load("List.CrossJoin"),
-    List.CrossJoin({"A","B"},{1..3})
-//Result: #table({"A","B"},{{"A",1},{"A",2},{"A",3},{"B",1},{"B",2},{"B",3}})
-*/
-
-let List.CrossJoin = (A as list, B as list) =>
-
-let
-	firstList = List.RemoveNulls(A),
-	secondList = List.RemoveNulls(B),
-	firstLength = List.Count(firstList),
-	secondLength = List.Count(secondList),
-	resultFirstList = List.Generate( () => 0, each _ < firstLength * secondLength, each _ + 1, each firstList{ Number.IntegerDivide(_, secondLength) }),
-	resultSecondList = List.Repeat( secondList, firstLength)
-in
-	Table.FromColumns({resultFirstList, resultSecondList})
-
-in List.CrossJoin
-
+/*
+//Returns the Cartesian product (i.e. crossjoin) of two arguments (can be lists or tables).
+//Usage:
+    List.CrossJoin = Load("List.CrossJoin"),
+    List.CrossJoin({"A","B"},{1..3})
+//Result: #table({"A","B"},{{"A",1},{"A",2},{"A",3},{"B",1},{"B",2},{"B",3}})
+*/
+
+(A as list, B as list) as table =>
+
+let
+	firstList = List.RemoveNulls(A),
+	secondList = List.RemoveNulls(B),
+	firstLength = List.Count(firstList),
+	secondLength = List.Count(secondList),
+	resultFirstList = List.Generate( () => 0, each _ < firstLength * secondLength, each _ + 1, each firstList{ Number.IntegerDivide(_, secondLength) }),
+	resultSecondList = List.Repeat( secondList, firstLength)
+in
+	Table.FromColumns({resultFirstList, resultSecondList})
+

List.FlatMap.m

@@ -0,0 +1,11 @@
+/*
+Maps a list using a lambda in such a way that all results are combined back into a single list.
+//Usage:
+let
+    List.FlatMap = Load("List.FlatMap")
+in
+    List.FlatMap({1,2,3}, (_) => List.Numbers(1, _))
+//Result: {1, 1, 2, 1, 2, 3}
+*/
+
+(lst as list, fn as function) => List.Combine(List.Transform(lst, fn))

List.ToRecord.m

@@ -0,0 +1,17 @@
+/*
+//Transform a list of strings to a record using a given lambda (passed values k)
+//The built-in Record.FromList only takes static lists...
+//Usage:
+let
+    List.ToRecord = Load("List.ToRecord"),
+    list = {"a","b"}
+in
+    List.ToRecord(list, (k) => Text.Upper(k))
+//Result: [a="A", b="B"]
+*/
+
+(List as list, Lambda as function) as record =>
+let
+    Transformed = List.Transform(List, Lambda)	//each Lambda(_)
+in
+    Record.FromList(Transformed, List)

List.Zip.m

@@ -0,0 +1,29 @@
+/*
+//Zip a list of lists so as to 'transpose' them -- as records if names are specified.
+//Usage:
+    List.Zip = Load("List.Zip"),
+    List.Zip({{1,2,3},{"a","b","c"}}, {"num","let"})
+//Result: {[num=1, let="a"],[num=2, let="b"],[num=3, let="c"]}
+*/
+
+(listOfLists as list, optional names as list) as list =>
+let
+	max = List.Max(List.Transform(listOfLists, each List.Count(_))),
+	zipped = List.Skip(List.Generate(
+        ()=>[
+            i = -1,
+			vals = {},
+			combined = {}
+        ],
+        each [i] < max,
+        each [
+            i = [i] + 1,
+            vals = List.Transform(listOfLists, each _{i}),
+			combined = if names = null then vals else Record.FromList(vals, names)
+        ],
+        each [combined]
+    )),
+	tablized = Table.FromRecords(zipped) //Table.FromRows
+in
+	tablized
+

Load.m

@@ -0,0 +1,51 @@
+/*
+Allows dynamically loading an M function from a text file (extension: .m) in a given folder for use in Power Query. This allows you to easily reuse  a set of functions in multiple workbooks without having to sync each change to all files using it.
+
+The point here is that by separating universally useful functions from an individual workbook, you will feel encouraged to use more modular code,  solving each common sub-problem only once, rather than remaining stuck in 'vanilla' M and resolving the same problems repeatedly.
+
+Moreover, coding this way will also further facilitate sharing code with other Power Query users, allowing for a more collaborative environment, gradually pushing forward the Power Query community as a whole.
+
+Nevertheless, if the function in question has already been imported into the workbook, the local copy will be used. This would allow you to either call the function locally right away, or Load() the existing function again.
+
+Using Load() would not only allow you to use functions in their intended naming conventions (i.e. Text.ReplaceAll rather than with the period replaced by an underscore), but would technically also allow you to add additional wrapper functions around your code, which could be used to enable persistent memoization (using say Redis) or code profiling calls... though presumably no-one has done this so far yet.
+
+Parameters:
+    fnName: name of the text file you wish to load without the .m extension
+    optional BasePath: the file path to look in for the text file; defaults to the path specified in the LoadPath query.
+
+Usage:
+    // loads the function Type.ToText from file 'Type.ToText.m' in the load path
+let
+    Type.ToText = Load("Type.ToText")
+in
+    Type.ToText(type {number})
+
+// Result: "list"
+
+Warning: this function may triggers a Formula.Firewall error for referencing both an external query (LoadPath) as well as external files.
+
+If you run into this, you can get around this by enabling the FastCombine option, in Power Query Options -> Privacy -> Fast Combine -> 'Ignore the 
+Privacy levels and potentially improve performance'.
+
+If you'd prefer not to do this however, you could also just replace the LoadPath reference below with a static absolute path reference.
+
+*/
+
+(fnName as text, optional BasePath as text) as function =>
+let
+    //If you wish to hardcode the path to load the queries from, you can edit the following line:
+    DefaultPath = LoadPath,
+    //DefaultPath = "D:\pquery",
+    BasePath = if (BasePath<>null) then BasePath else DefaultPath,
+	Path = BasePath & (if Text.End(BasePath, 1) <> "\" then "\" else ""),
+    File = Path & fnName & ".m",
+    AltFile = Path & Text.Replace(fnName, "_", ".") & ".m",  //just in case...
+    Source = Text.FromBinary(Binary.Buffer(
+		try File.Contents(File)
+		otherwise File.Contents(AltFile)
+	)),
+    Function = try Expression.Evaluate(Text.Replace(fnName, ".", "_"), #shared)  //if already imported into the workbook just use the existing one
+		otherwise Expression.Evaluate(Source, #shared)	//if not imported yet try loading it from the text file in the folder
+in
+    Function
+

LoadPath.m

@@ -0,0 +1,6 @@
+let
+    //Feel free to just put your own fixed path in here instead of using the table from this workbook!
+    //Source = "D:\pquery\"
+    Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content]{0}[Path]
+in
+    Source

M.UDFs.m

@@ -0,0 +1,6 @@
+let
+    Source = Record.FieldNames(#shared),
+    UDFs = List.Select(Source, each Record.HasFields(#sections[Section1], _)),
+    Return = UDFs
+in
+    Return

M.constants.m

@@ -0,0 +1,6 @@
+let
+    Source = M_library,
+    Constants = Table.SelectRows(Source, each not Value.Is([Value], type type) and not Value.Is([Value], type function)),
+    Return = Constants
+in
+    Return

M.functions.m

@@ -0,0 +1,23 @@
+let
+    Type.ToText = Load("Type.ToText"),
+    Value.ToText = Load("Value.ToText"),
+    Text.Count = Load("Text.Count"),
+    Source = M_library,
+    Functions = Table.SelectRows(Source, each Value.Is([Value], type function)),
+    AddFType = Table.AddColumn(Functions, "FType", each Value.Type([Value])),
+    AddReturn = Table.AddColumn(AddFType, "Returns", each Type.FunctionReturn([FType])),
+    AddRetTxt = Table.AddColumn(AddReturn, "RetText", each Type.ToText([Returns])),
+    AddRetRec = Table.AddColumn(AddRetTxt, "RetRecursive", each Type.ToText([Returns], true)),
+    AddRetNull = Table.AddColumn(AddRetRec, "RetNullable", each Type.IsNullable([Returns])),
+    AddRetType = Table.AddColumn(AddRetNull, "RetType", each let Type = Type.ToText(Type.NonNullable([Returns])) in if Type = "anynonnull" and [RetNullable] then "any" else Type),
+    AddParams = Table.AddColumn(AddRetType, "Parameters", each Type.FunctionParameters([FType])),
+    AddNumPars = Table.AddColumn(AddParams, "NumParams", each Record.FieldCount([Parameters])),
+    AddReqd = Table.AddColumn(AddNumPars, "Required", each Type.FunctionRequiredParameters([FType])),
+    AddSig = Table.AddColumn(AddReqd, "Signature", each Value.ToText([Value])),
+    AddSigRec = Table.AddColumn(AddSig, "SigRecursive", each [Name] & " => " & Value.ToText([Value], true)),
+    AddTally = Table.AddColumn(AddSigRec, "Times Used", each Text.Count(Text_Queries, [Name])),
+    GoodCols = Table.RemoveColumns(AddTally, {"Type", "TypeRecurs"}),
+    Return = GoodCols
+in
+    Return
+

M.library.m

@@ -0,0 +1,26 @@
+let
+    Source = Table.Sort(Record.ToTable(#shared),{{"Name", Order.Ascending}}),
+    Categorized = Table.AddColumn(Source, "Status", each if Record.HasFields(#sections[Section1], [Name]) then "User defined" else "Built in"),
+    // Annoying I need to filter out user defined stuff, but this resolves a cyclic reference caused if both F and this refer to all custom functions (which includes each other)
+    Filtered = Table.SelectRows(Categorized, each [Status] = "Built in"),
+    AddType = Table.AddColumn(Filtered, "Type", each Value_TypeToText([Value])),
+    AddTypeRec = Table.AddColumn(AddType, "TypeRecurs", each Value_TypeToText([Value], true)),
+    AddCat = Table.AddColumn(AddTypeRec, "Category", each
+    let
+        cut = Text.Split(Text.Replace([Name],"_","."),".")
+    in
+        (try
+            if
+                List.Contains({"Database", "Type"}, cut{1})
+            then
+                cut{1}
+            else
+                cut{0}
+        otherwise
+            "Custom"
+        )
+    ),
+    Return = AddCat
+in
+    Return
+

M.types.m

@@ -0,0 +1,6 @@
+let
+    Source = M_library,
+    Types = Table.SelectRows(Source, each Value.Is([Value], type type)),
+    Return = Types
+in
+    Return