src: initial localization support

This is a work in progress effort to introduce the *optional*
ability to localize node's own outputs. Currently, Node (and V8)
outputs english only error, debug and help output. This is a step
towards allowing translated versions of node to be built.

Currently, this is hidden behind the --with-intl=full-icu switch.
This is because there are certain capabilities of ICU that are
only enabled with the full data set.

This is a work in progress that SHOULD NOT BE LANDED in master yet.

Makefile

@@ -70,6 +70,7 @@ clean:
 	@if [ -d out ]; then find out/ -name '*.o' -o -name '*.a' -o -name '*.d' | xargs rm -rf; fi
 	-rm -rf node_modules
 	@if [ -d deps/icu ]; then echo deleting deps/icu; rm -rf deps/icu; fi
+	-rm -rf out/Release/obj/gen/noderestmp
 	-rm -f test.tap
 
 distclean:

deps/l10n/README.md

@@ -0,0 +1,43 @@
+Localization bundle for Node.
+
+This is pretty straightforward... if ICU is present and --with-intl-full-icu,
+then ICU's resource bundle mechanism is used, the resources are compiled
+statically into the library, which can then be used within Node. If ICU is not
+present, a simple printf fallback is used.
+
+```
+./configure --with-intl=full-icu
+make
+```
+When the --with-intl=full-icu switch is on, the resources are compiled into a
+static library that is statically linked into Node. The next step will be to
+make it so that additional bundles can be specified at runtime.
+
+Resource bundles are located in the resources directory. Standard ICU bundle
+format but keep it simple, we currently only support top level resources.
+
+Within the C/C++ code, use the macros:
+
+```cc
+#include "node_l10n.h"
+
+L10N_PRINT("TEST", "This is the fallback");
+L10N_PRINTF("TEST2", "Testing %s %d", "a", 1);
+```
+
+In the JS code, use the internal/l10n.js
+```javascript
+const l10n = require('internal/l10n');
+console.log(l10n("TEST", "This is the fallback"));
+console.log(l10n("TEST2", "Fallback %s %d", "a", 1));
+```
+
+Use the `--icu-data-dir` switch to specify a location containing alternative
+node.dat files containing alternative translations. Note that this is the
+same switch used to specify alternative ICU common data files.
+
+One approach that ought to work here is publishing translation node.dat files
+to npm. Then, the developer does a `npm install node_dat_de` (for instance)
+to grab the appropriate translations. Then, then can start node like:
+
+`node --icu-data-dir=node_modules/node_dat_de` and have things just work.

deps/l10n/icures.py

@@ -0,0 +1,129 @@
+#!/usr/bin/python
+
+import sys
+import shutil
+reload(sys)
+sys.setdefaultencoding("utf-8")
+
+import optparse
+import os
+import glob
+
+endian=sys.byteorder
+
+parser = optparse.OptionParser(usage="usage: %prog -n {NAME} -d {DEST} -i {ICU}")
+
+parser.add_option("-d", "--dest-dir",
+                  action="store",
+                  dest="dest",
+                  help="The destination directory")
+
+parser.add_option("-n", "--name",
+                  action="store",
+                  dest="name",
+                  help="The application package name")
+
+parser.add_option("-i", "--icu-path",
+                  action="store",
+                  dest="icu",
+                  help="The ICU tool path")
+
+parser.add_option("-l", "--endian",
+                  action="store",
+                  dest="endian",
+                  help='endian: big, little or host. your default is "%s"' % endian, default=endian, metavar='endianess')
+
+(options, args) = parser.parse_args()
+
+optVars = vars(options);
+
+for opt in ["dest", "name", "icu"]:
+    if optVars[opt] is None:
+        parser.error("Missing required option: %s" % opt)
+        sys.exit(1)
+
+if options.endian not in ("big", "little", "host"):
+    parser.error("Unknown endianess: %s" % options.endian)
+    sys.exit(1)
+
+if options.endian == "host":
+    options.endian = endian
+
+if not os.path.isdir(options.dest):
+    parser.error("Destination is not a directory")
+    sys.exit(1)
+
+if options.icu[-1] is '"':
+    options.icu = options.icu[:-1]
+
+if not os.path.isdir(options.icu):
+    parser.error("ICU Path is not a directory")
+    sys.exit(1)
+
+if options.icu[-1] != os.path.sep:
+    options.icu += os.path.sep
+
+genrb = options.icu + 'genrb'
+icupkg = options.icu + 'icupkg'
+
+if sys.platform.startswith('win32'):
+    genrb += '.exe'
+    icupkg += '.exe'
+
+if not os.path.isfile(genrb):
+    parser.error('ICU Tool "%s" does not exist' % genrb)
+    sys.exit(1)
+
+if not os.path.isfile(icupkg):
+    parser.error('ICU Tool "%s" does not exist' % icupkg)
+    sys.exit(1)
+
+def runcmd(tool, cmd, doContinue=False):
+    cmd = "%s %s" % (tool, cmd)
+    rc = os.system(cmd)
+    if rc is not 0 and not doContinue:
+        print "FAILED: %s" % cmd
+        sys.exit(1)
+    return rc
+
+resfiles = glob.glob("%s%s*.res" % (options.dest, os.path.sep))
+_listfile = os.path.join(options.dest, 'packagefile.lst')
+datfile = "%s%s%s.dat" % (options.dest, os.path.sep, options.name)
+
+def clean():
+    for f in resfiles:
+        if os.path.isfile(f):
+            os.remove(f)
+    if os.path.isfile(_listfile):
+        os.remove(_listfile)
+    if (os.path.isfile(datfile)):
+        os.remove(datfile)
+
+## Step 0, Clean up from previous build
+clean()
+
+## Step 1, compile the txt files in res files
+
+if sys.platform.startswith('win32'):
+  srcfiles = glob.glob('resources/*.txt')
+  runcmd(genrb, "-e utf16 -d %s %s" % (options.dest, " ".join(srcfiles)))
+else:
+  runcmd(genrb, "-e utf16 -d %s resources%s*.txt" % (options.dest, os.path.sep))
+
+resfiles = [os.path.relpath(f) for f in glob.glob("%s%s*.res" % (options.dest, os.path.sep))]
+
+# pkgdata requires relative paths... it's annoying but necessary
+# for us to change into the dest directory to work
+cwd = os.getcwd();
+os.chdir(options.dest)
+
+## Step 2, generate the package list
+listfile = open(_listfile, 'w')
+listfile.write(" ".join([os.path.basename(f) for f in resfiles]))
+listfile.close()
+
+## Step 3, generate the dat file using icupkg and the package list
+runcmd(icupkg, '-a packagefile.lst new %s.dat' % options.name);
+
+## All done with this tool at this point...
+os.chdir(cwd); # go back to original working directory

deps/l10n/include/l10n.h

@@ -0,0 +1,35 @@
+#ifndef L10N__H
+#define L10N__H
+
+#include <unicode/uloc.h>
+#include <unicode/ustdio.h>
+#include <unicode/ustring.h>
+
+#ifdef _WIN32
+# define L10N_EXTERN /* nothing */
+#elif __GNUC__ >= 4
+# define L10N_EXTERN __attribute__((visibility("default")))
+#else
+# define L10N_EXTERN /* nothing */
+#endif
+
+/**
+ * Initialize the resource bundle. This will register an atexit handler
+ * to deal with the cleanup in normal termination
+ **/
+L10N_EXTERN bool l10n_initialize(const char * locale, const char * icu_data_dir);
+
+/**
+ * Lookup the key, return the value. Doesn't get much easier. If the key
+ * is not found in the bundle, fallback is returned instead. The caller
+ * owns the string and must delete[] it when done lest horrible things.
+ **/
+L10N_EXTERN const uint16_t * l10n_fetch(const char * key,
+                                        const uint16_t * fallback);
+
+#define L10N(key, fallback) l10n_fetch(key, fallback)
+#define L10N_LOCALE() uloc_getDefault()
+#define L10N_INIT(locale, icu_data_dir)                                       \
+  do {l10n_initialize(locale, icu_data_dir);} while(0)
+
+#endif // L10N__H

deps/l10n/l10n.gyp

@@ -0,0 +1,123 @@
+{
+  'includes': [ '../../icu_config.gypi' ],
+  'targets': [
+    {
+      'target_name': 'noderes',
+      'type': '<(library)',
+      'conditions': [
+        [
+          'v8_enable_i18n_support==1',
+          {
+            'include_dirs': [ 'include', 'src' ],
+            'direct_dependent_settings': {
+              'include_dirs': [ 'include' ]
+            },
+            'sources': [
+              'include/l10n.h',
+              'include/l10n_version.h',
+              'src/l10n.cc'
+            ],
+            'dependencies': [
+              '<(icu_gyp_path):icui18n',
+              '<(icu_gyp_path):icuuc',
+              'icu_noderes_data'
+            ]
+          }
+        ]
+      ]
+    },
+    {
+      #### build the resource bundle using ICU's tools ####
+      'target_name': 'icu_noderes_data',
+      'type': '<(library)',
+      'conditions': [
+        [
+          'v8_enable_i18n_support==1',
+          {
+            'toolsets': [ 'target' ],
+            'dependencies': [
+              '<(icu_gyp_path):icu_implementation#host',
+              '<(icu_gyp_path):icu_uconfig',
+              '<(icu_gyp_path):genrb#host',
+              '<(icu_gyp_path):genccode#host',
+              '<(icu_gyp_path):icupkg#host'
+            ],
+            'actions': [
+              {
+                'action_name': 'icures',
+                'inputs': [
+                  'resources/root.txt',
+                  'resources/en.txt',
+                  'resources/en_US.txt'
+                ],
+                'outputs': [
+                  '<(SHARED_INTERMEDIATE_DIR)/noderestmp/node.dat'
+                ],
+                'action': [
+                  'python',
+                  'icures.py',
+                  '-n', 'node',
+                  '-d', '<(SHARED_INTERMEDIATE_DIR)/noderestmp',
+                  '-i', '<(PRODUCT_DIR)'
+                ]
+              },
+              {
+                'action_name': 'icugen',
+                'inputs': [
+                  '<(SHARED_INTERMEDIATE_DIR)/noderestmp/node.dat'
+                ],
+                'conditions': [
+                  [
+                    'OS != "win"',
+                    {
+                      'outputs': [
+                        '<(SHARED_INTERMEDIATE_DIR)/node_dat.c'
+                      ],
+                      'action': [
+                        '<(PRODUCT_DIR)/genccode',
+                        '-e', 'node',
+                        '-d', '<(SHARED_INTERMEDIATE_DIR)',
+                        '-f', 'node_dat',
+                        '<@(_inputs)'
+                      ]
+                    },
+                    {
+                      'outputs': [
+                        '<(SHARED_INTERMEDIATE_DIR)/node_dat.obj'
+                      ],
+                      'action': [
+                        '<(PRODUCT_DIR)/genccode',
+                        '-o',
+                        '-d', '<(SHARED_INTERMEDIATE_DIR)',
+                        '-n', 'node',
+                        '-e', 'node',
+                        '<@(_inputs)'
+                      ]
+                    }
+                  ]
+                ]
+              }
+            ],
+            'conditions': [
+              [ 'OS == "win"',
+                {
+                  'sources': [
+                    '<(SHARED_INTERMEDIATE_DIR)/node_dat.obj'
+                  ]
+                },
+                {
+                  'include_dirs': [
+                    '../icu/source/common'
+                  ],
+                  'sources': [
+                    '<(SHARED_INTERMEDIATE_DIR)/node_dat.c'
+                  ]
+                }
+              ]
+            ]
+          }
+        ]
+      ]
+    }
+  ]
+}

deps/l10n/src/l10n.cc

@@ -0,0 +1,50 @@
+#include <stdlib.h>
+#include <unicode/udata.h>
+#include <unicode/ures.h>
+#include "l10n.h"
+
+// The ICU bundle name
+#define L10N_APPDATA "node"
+
+extern "C" const char U_DATA_API node_dat[];
+
+UResourceBundle *bundle;
+
+void l10n_cleanup() {
+  ures_close(bundle);
+}
+
+bool l10n_initialize(const char * locale,
+                     const char * icu_data_dir) {
+  UErrorCode status = U_ZERO_ERROR;
+  if (!icu_data_dir) {
+    udata_setAppData("node", &node_dat, &status);
+  }
+  if (U_FAILURE(status)) {
+    return FALSE;
+  } else {
+    bundle = ures_open(L10N_APPDATA, locale, &status);
+    if (U_SUCCESS(status)) {
+      atexit(l10n_cleanup);
+      return TRUE;
+    } else {
+      return FALSE;
+    }
+  }
+}
+
+const uint16_t * l10n_fetch(const char * key,
+                            const uint16_t * fallback) {
+  UErrorCode status = U_ZERO_ERROR;
+  int32_t len = 0;
+  const UChar* res = ures_getStringByKey(
+                          bundle,
+                          key,
+                          &len,
+                          &status);
+  const uint16_t* ret = static_cast<const uint16_t*>(res);
+  if (U_SUCCESS(status)) {
+    return ret;
+  }
+  return fallback;
+}