Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: error on empty array #57

Open
wants to merge 22 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
9e9642f
Added ZipWriter for in-memory processing of compressesed shapefiles
Mar 20, 2019
32d1c51
Decrease default buffer size
atsibulnik Apr 13, 2019
cc37aaf
Update GCS WKT for WGS84
Apr 16, 2019
a672c6c
Proj4 WKT for NAD27
May 13, 2019
df08c3e
Use non-default CRS WKT
May 13, 2019
3c1358e
Support for multilayer shapefiles added
May 20, 2019
b476d62
Update crs wkt strings
Jun 2, 2019
6dbca2a
fix(zipReader): filter hidden dir and __MACOSX dir to avoir duplicate…
geoduf Feb 17, 2021
73e65b5
Merge pull request #1 from Karnott/fix/hidden-dir-zip-file
lborie Feb 17, 2021
71ea197
Add empty shapefile layers support
May 26, 2021
0cf0db7
Check for errors while reading header
peterstace Mar 29, 2019
09c9a6e
Add missing errReader usage
peterstace Mar 29, 2019
96525ff
fix(zipReader): filter hidden dir and __MACOSX dir to avoir duplicate…
geoduf Feb 17, 2021
0a33eeb
feat: github action
lborie Apr 13, 2023
5f89b6b
Merge pull request #3 from Karnott/github_action
lborie Apr 13, 2023
0c51c2e
Merge pull request #2 from Karnott/atsilbunik_rebase
lborie Apr 13, 2023
e9315b1
fix: adapt the current documentation to karnott organisation
lborie Apr 13, 2023
e233bf1
Merge pull request #4 from Karnott/documentation
lborie Apr 13, 2023
cdb4b35
fix(sequentialreader): fix panic because of nil pointer
geoduf Jun 27, 2024
eee439f
Merge pull request #5 from Karnott/fix/close-panic
geoduf Jun 27, 2024
ebda50a
fix(sequentialreader): remove ioutil lib + fix warnings
geoduf Aug 2, 2024
d147335
fix(sequentialreader): check size of dbfrow before reading index 0
geoduf Aug 2, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
name: Go package

on: [push]

jobs:
build:

runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 'stable'

- name: Build
run: go build -v ./...
13 changes: 4 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,22 +1,17 @@
go-shp
======

[![Build Status](https://travis-ci.org/jonas-p/go-shp.svg?branch=master)](https://travis-ci.org/jonas-p/go-shp)
[![Build status](https://ci.appveyor.com/api/projects/status/b64sntax4kxlouxa?svg=true)](https://ci.appveyor.com/project/fawick/go-shp)
[![Go Report Card](https://goreportcard.com/badge/github.com/jonas-p/go-shp)](https://goreportcard.com/report/github.com/jonas-p/go-shp)
[![Codevov](https://codecov.io/gh/jonas-p/go-shp/branch/master/graphs/badge.svg)](https://codecov.io/gh/jonas-p/go-shp)
![Build status](https://github.com/karnott/go-shp/actions/workflows/build.yml/badge.svg)
[![Go Report Card](https://goreportcard.com/badge/github.com/karnott/go-shp)](https://goreportcard.com/report/github.com/jonas-p/go-shp)

Go library for reading and writing ESRI Shapefiles. This is a pure Golang implementation based on the ESRI Shapefile technical description.

### Usage
#### Installation

go get github.com/jonas-p/go-shp

#### Importing

```go
import "github.com/jonas-p/go-shp"
import "github.com/karnott/go-shp"
```

### Examples
Expand Down Expand Up @@ -83,5 +78,5 @@ for n, point := range points {

### Resources

- [Documentation on godoc.org](http://godoc.org/github.com/jonas-p/go-shp)
- [Documentation on godoc.org](http://godoc.org/github.com/karnott/go-shp)
- [ESRI Shapefile Technical Description](http://www.esri.com/library/whitepapers/pdfs/shapefile.pdf)
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module github.com/karnott/go-shp

go 1.20
42 changes: 23 additions & 19 deletions sequentialreader.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"encoding/binary"
"fmt"
"io"
"io/ioutil"
"math"
"strings"
)
Expand Down Expand Up @@ -81,19 +80,19 @@ func (sr *seqReader) readHeaders() {

er := &errReader{Reader: sr.shp}
// shp headers
io.CopyN(ioutil.Discard, er, 24)
io.CopyN(io.Discard, er, 24)
var l int32
binary.Read(er, binary.BigEndian, &l)
sr.filelength = int64(l) * 2
io.CopyN(ioutil.Discard, er, 4)
io.CopyN(io.Discard, er, 4)
binary.Read(er, binary.LittleEndian, &sr.geometryType)
sr.bbox.MinX = readFloat64(er)
sr.bbox.MinY = readFloat64(er)
sr.bbox.MaxX = readFloat64(er)
sr.bbox.MaxY = readFloat64(er)
io.CopyN(ioutil.Discard, er, 32) // skip four float64: Zmin, Zmax, Mmin, Max
io.CopyN(io.Discard, er, 32) // skip four float64: Zmin, Zmax, Mmin, Max
if er.e != nil {
sr.err = fmt.Errorf("Error when reading SHP header: %v", er.e)
sr.err = fmt.Errorf("error when reading SHP header: %v", er.e)
return
}

Expand All @@ -102,22 +101,22 @@ func (sr *seqReader) readHeaders() {
if sr.dbf == nil {
return
}
io.CopyN(ioutil.Discard, er, 4)
io.CopyN(io.Discard, er, 4)
binary.Read(er, binary.LittleEndian, &sr.dbfNumRecords)
binary.Read(er, binary.LittleEndian, &sr.dbfHeaderLength)
binary.Read(er, binary.LittleEndian, &sr.dbfRecordLength)
io.CopyN(ioutil.Discard, er, 20) // skip padding
io.CopyN(io.Discard, er, 20) // skip padding
numFields := int(math.Floor(float64(sr.dbfHeaderLength-33) / 32.0))
sr.dbfFields = make([]Field, numFields)
binary.Read(er, binary.LittleEndian, &sr.dbfFields)
buf := make([]byte, 1)
er.Read(buf[:])
if er.e != nil {
sr.err = fmt.Errorf("Error when reading DBF header: %v", er.e)
sr.err = fmt.Errorf("error when reading DBF header: %v", er.e)
return
}
if buf[0] != 0x0d {
sr.err = fmt.Errorf("Field descriptor array terminator not found")
sr.err = fmt.Errorf("field descriptor array terminator not found")
return
}
sr.dbfRow = make([]byte, sr.dbfRecordLength)
Expand All @@ -139,7 +138,7 @@ func (sr *seqReader) Next() bool {

if er.e != nil {
if er.e != io.EOF {
sr.err = fmt.Errorf("Error when reading shapefile header: %v", er.e)
sr.err = fmt.Errorf("error when reading shapefile header: %v", er.e)
} else {
sr.err = io.EOF
}
Expand All @@ -149,7 +148,7 @@ func (sr *seqReader) Next() bool {
var err error
sr.shape, err = newShape(shapetype)
if err != nil {
sr.err = fmt.Errorf("Error decoding shape type: %v", err)
sr.err = fmt.Errorf("error decoding shape type: %v", err)
return false
}
sr.shape.read(er)
Expand All @@ -160,25 +159,25 @@ func (sr *seqReader) Next() bool {
// iterating over all shapes.
er.e = nil
case er.e != nil:
sr.err = fmt.Errorf("Error while reading next shape: %v", er.e)
sr.err = fmt.Errorf("error while reading next shape: %v", er.e)
return false
}
skipBytes := int64(size)*2 + 8 - er.n
_, ce := io.CopyN(ioutil.Discard, er, skipBytes)
_, ce := io.CopyN(io.Discard, er, skipBytes)
if er.e != nil {
sr.err = er.e
return false
}
if ce != nil {
sr.err = fmt.Errorf("Error when discarding bytes on sequential read: %v", ce)
sr.err = fmt.Errorf("error when discarding bytes on sequential read: %v", ce)
return false
}
if _, err := io.ReadFull(sr.dbf, sr.dbfRow); err != nil {
sr.err = fmt.Errorf("Error when reading DBF row: %v", err)
sr.err = fmt.Errorf("error when reading DBF row: %v", err)
return false
}
if sr.dbfRow[0] != 0x20 && sr.dbfRow[0] != 0x2a {
sr.err = fmt.Errorf("Attribute row %d starts with incorrect deletion indicator", num)
if (len(sr.dbfRow) == 0) || (sr.dbfRow[0] != 0x20 && sr.dbfRow[0] != 0x2a) {
sr.err = fmt.Errorf("attribute row %d starts with incorrect deletion indicator", num)
}
return sr.err == nil
}
Expand Down Expand Up @@ -212,8 +211,13 @@ func (sr *seqReader) Err() error {

// Close closes the seqReader and free all the allocated resources.
func (sr *seqReader) Close() error {
if err := sr.shp.Close(); err != nil {
return err
if sr.shp != nil {
if err := sr.shp.Close(); err != nil {
return err
}
}
if sr.dbf == nil {
return nil
}
if err := sr.dbf.Close(); err != nil {
return err
Expand Down
54 changes: 53 additions & 1 deletion shapefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package shp

import (
"encoding/binary"
"errors"
"io"
"strings"
)
Expand Down Expand Up @@ -29,6 +30,32 @@ const (
MULTIPATCH ShapeType = 31
)

var shapeTypeNameMap = map[string]ShapeType{
"NULL": NULL,
"POINT": POINT,
"POLYLINE": POLYLINE,
"POLYGON": POLYGON,
"MULTIPOINT": MULTIPOINT,
"POINTZ": POINTZ,
"POLYLINEZ": POLYLINEZ,
"POLYGONZ": POLYGONZ,
"MULTIPOINTZ": MULTIPOINTZ,
"POINTM": POINTM,
"POLYLINEM": POLYLINEM,
"POLYGONM": POLYGONM,
"MULTIPOINTM": MULTIPOINTM,
"MULTIPATCH": MULTIPATCH,
}

// ParseShapeType tries to extract ShapeType from string
func ParseShapeType(typ string) (ShapeType, error) {
t, ok := shapeTypeNameMap[strings.ToUpper(typ)]
if !ok {
return NULL, errors.New("invalid shape type")
}
return t, nil
}

// Box structure made up from four coordinates. This type
// is used to represent bounding boxes
type Box struct {
Expand Down Expand Up @@ -571,6 +598,23 @@ type Field struct {
Padding [14]byte
}

const (
// TextFieldLength is default length for 'Text' geodatabase data type
TextFieldLength uint8 = 254
// FloatFieldLength is default length for 'Float' geodatabase data type
FloatFieldLength uint8 = 18
// FloatFieldPrecision is default precision for 'Float' geodatabase data type
FloatFieldPrecision uint8 = 8
// ShortFieldLength is default length for 'Short' geodatabase data type
ShortFieldLength uint8 = 10
// LongFieldLength is default length for 'Long' geodatabase data type
LongFieldLength uint8 = 18
// DateFieldLength is default length for 'Date' geodatabase data type
DateFieldLength uint8 = 8
// BoolFieldLength is default length for boolean data type
BoolFieldLength uint8 = 3
)

// Returns a string representation of the Field. Currently
// this only returns field name.
func (f Field) String() string {
Expand Down Expand Up @@ -606,7 +650,15 @@ func FloatField(name string, length uint8, precision uint8) Field {
// DBF file. Used to store Date strings formatted as YYYYMMDD. Data wise this
// is the same as a StringField with length 8.
func DateField(name string) Field {
field := Field{Fieldtype: 'D', Size: 8}
field := Field{Fieldtype: 'D', Size: DateFieldLength}
copy(field.Name[:], []byte(name))
return field
}

// BoolField feturns a Field that can be used in SetFields to initialize the
// DBF file. Used to store boolean values using "Yes"/"No" strings
func BoolField(name string) Field {
field := Field{Fieldtype: 'C', Size: BoolFieldLength}
copy(field.Name[:], name)
return field
}
4 changes: 3 additions & 1 deletion zipreader.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"io"
"path"
"regexp"
"strings"
)

Expand Down Expand Up @@ -71,8 +72,9 @@ func ShapesInZip(zipFilePath string) ([]string, error) {

func shapesInZip(z *zip.ReadCloser) []*zip.File {
var shapeFiles []*zip.File
hiddenDirRegex := regexp.MustCompile("(?i)(__MACOSX|^\\.|\\/\\.)")
for _, f := range z.File {
if strings.HasSuffix(f.Name, ".shp") {
if strings.HasSuffix(f.Name, ".shp") && !hiddenDirRegex.MatchString(f.Name) {
shapeFiles = append(shapeFiles, f)
}
}
Expand Down
Loading