increase size of index, line, and column fields (#310)

* increase size of index, line, and column fields

* use size_t instead of unsigned long long

* better test infrastructure for test for large file

* only run large file test when env var is set

* fix review comments regarding env vars

* fix missing import on python 3

* force all tests in CI

.appveyor.yml

@@ -11,6 +11,8 @@ image:
 environment:
   libyaml_repo_url: https://github.com/yaml/libyaml.git
   libyaml_refspec: 0.2.2
+  PYYAML_TEST_GROUP: all
+
 #  matrix:
 #    - PYTHON_VER: Python27
 #    - PYTHON_VER: Python27-x64

.travis.yml

@@ -4,6 +4,10 @@ language: python
 
 cache: pip
 
+env:
+  global:
+    - PYYAML_TEST_GROUP=all
+
 matrix:
   include:
     - python: 2.7

ext/_yaml.pyx

@@ -63,13 +63,13 @@ MappingNode = yaml.nodes.MappingNode
 
 cdef class Mark:
     cdef readonly object name
-    cdef readonly int index
-    cdef readonly int line
-    cdef readonly int column
+    cdef readonly size_t index
+    cdef readonly size_t line
+    cdef readonly size_t column
     cdef readonly buffer
     cdef readonly pointer
 
-    def __init__(self, object name, int index, int line, int column,
+    def __init__(self, object name, size_t index, size_t line, size_t column,
             object buffer, object pointer):
         self.name = name
         self.index = index

tests/lib/test_yaml_ext.py

@@ -1,6 +1,6 @@
 
 import _yaml, yaml
-import types, pprint
+import types, pprint, tempfile, sys, os
 
 yaml.PyBaseLoader = yaml.BaseLoader
 yaml.PySafeLoader = yaml.SafeLoader
@@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False):
 test_c_emitter.unittest = ['.data', '.canonical']
 test_c_emitter.skip = ['.skip-ext']
 
+def test_large_file(verbose=False):
+    SIZE_LINE = 24
+    SIZE_ITERATION = 0
+    SIZE_FILE = 31
+    if sys.maxsize <= 2**32:
+        return
+    if os.environ.get('PYYAML_TEST_GROUP', '') != 'all':
+        return
+    with tempfile.TemporaryFile() as temp_file:
+        for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1):
+            temp_file.write(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION))
+        temp_file.seek(0)
+        yaml.load(temp_file, Loader=yaml.CLoader)
+
+test_large_file.unittest = None
+
 def wrap_ext_function(function):
     def wrapper(*args, **kwds):
         _set_up()

tests/lib3/test_yaml_ext.py

@@ -1,6 +1,6 @@
 
 import _yaml, yaml
-import types, pprint
+import types, pprint, tempfile, sys, os
 
 yaml.PyBaseLoader = yaml.BaseLoader
 yaml.PySafeLoader = yaml.SafeLoader
@@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False):
 test_c_emitter.unittest = ['.data', '.canonical']
 test_c_emitter.skip = ['.skip-ext']
 
+def test_large_file(verbose=False):
+    SIZE_LINE = 24
+    SIZE_ITERATION = 0
+    SIZE_FILE = 31
+    if sys.maxsize <= 2**32:
+        return
+    if os.environ.get('PYYAML_TEST_GROUP', '') != 'all':
+        return
+    with tempfile.TemporaryFile() as temp_file:
+        for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1):
+            temp_file.write(bytes(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION), 'utf-8'))
+        temp_file.seek(0)
+        yaml.load(temp_file, Loader=yaml.CLoader)
+
+test_large_file.unittest = None
+
 def wrap_ext_function(function):
     def wrapper(*args, **kwds):
         _set_up()