Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit dcbc0ec

Browse files
authored
complete fixed-length fileformat and wire into omni extension v21 (#119)
1 parent 3cb6222 commit dcbc0ec

File tree

5 files changed

+106
-10
lines changed

5 files changed

+106
-10
lines changed

‎extensions/omniv21/fileformat/fixedlength/format.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,9 +129,8 @@ func (f *fixedLengthFileFormat) validateColumns(cols []*columnDecl) error {
129129

130130
func (f *fixedLengthFileFormat) CreateFormatReader(
131131
name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
132-
// TODO
133-
_ = runtime.(*fixedLengthFormatRuntime)
134-
return nil, nil
132+
rt := runtime.(*fixedLengthFormatRuntime)
133+
return NewReader(name, r, rt.Decl, rt.XPath)
135134
}
136135

137136
func (f *fixedLengthFileFormat) FmtErr(format string, args ...interface{}) error {

‎extensions/omniv21/fileformat/fixedlength/format_test.go‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
package fixedlength
22

33
import (
4+
"io"
45
"strings"
56
"testing"
67

78
"github.com/bradleyjkemp/cupaloy"
89
"github.com/jf-tech/go-corelib/jsons"
910
"github.com/jf-tech/go-corelib/strs"
11+
"github.com/jf-tech/go-corelib/testlib"
1012
"github.com/stretchr/testify/assert"
1113

1214
"github.com/jf-tech/omniparser/errs"
1315
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
16+
"github.com/jf-tech/omniparser/idr"
1417
)
1518

1619
func TestValidateSchema(t *testing.T) {
@@ -304,10 +307,29 @@ func TestValidateSchema(t *testing.T) {
304307
}
305308

306309
func TestCreateFormatReader(t *testing.T) {
307-
_, err := NewFixedLengthFileFormat("test").CreateFormatReader(
308-
"test-input",
309-
strings.NewReader("TODO"),
310-
&fixedLengthFormatRuntime{})
310+
r, err := NewFixedLengthFileFormat("test").CreateFormatReader(
311+
"test",
312+
strings.NewReader("abcd\n1234\n"),
313+
&fixedLengthFormatRuntime{
314+
Decl: &fileDecl{
315+
Envelopes: []*envelopeDecl{
316+
{
317+
Name: strs.StrPtr("env1"),
318+
ByRows: testlib.IntPtr(2),
319+
Columns: []*columnDecl{
320+
{Name: "letters", StartPos: 1, Length: 3, LinePattern: strs.StrPtr("^[a-z]")},
321+
{Name: "numerics", StartPos: 1, Length: 3, LinePattern: strs.StrPtr("^[0-9]")},
322+
},
323+
},
324+
},
325+
},
326+
})
327+
assert.NoError(t, err)
328+
n, err := r.Read()
311329
assert.NoError(t, err)
312-
// TODO
330+
assert.Equal(t, `{"letters":"abc","numerics":"123"}`, idr.JSONify2(n))
331+
r.Release(n)
332+
n, err = r.Read()
333+
assert.Equal(t, io.EOF, err)
334+
assert.Nil(t, n)
313335
}

‎extensions/omniv21/fileformat/fixedlength/reader.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package fixedlength
22

33
import (
44
"bufio"
5+
"errors"
56
"fmt"
67
"io"
8+
"strings"
79

810
"github.com/antchfx/xpath"
911
"github.com/jf-tech/go-corelib/caches"
@@ -170,6 +172,42 @@ readEnvelope:
170172
return node, err
171173
}
172174

175+
func (r *reader) Release(n *idr.Node) {
176+
if r.target == n {
177+
r.target = nil
178+
}
179+
idr.RemoveAndReleaseTree(n)
180+
}
181+
182+
func (r *reader) IsContinuableError(err error) bool {
183+
return !IsErrInvalidEnvelope(err) && err != io.EOF
184+
}
185+
186+
func (r *reader) FmtErr(format string, args ...interface{}) error {
187+
return errors.New(r.fmtErrStr(format, args...))
188+
}
189+
173190
func (r *reader) fmtErrStr(format string, args ...interface{}) string {
174191
return fmt.Sprintf("input '%s' line %d: %s", r.inputName, r.line, fmt.Sprintf(format, args...))
175192
}
193+
194+
// NewReader creates an FormatReader for fixed-length file format.
195+
func NewReader(inputName string, r io.Reader, decl *fileDecl, xpathStr string) (*reader, error) {
196+
var expr *xpath.Expr
197+
var err error
198+
xpathStr = strings.TrimSpace(xpathStr)
199+
if xpathStr != "" && xpathStr != "." {
200+
expr, err = caches.GetXPathExpr(xpathStr)
201+
if err != nil {
202+
return nil, fmt.Errorf("invalid xpath '%s', err: %s", xpathStr, err.Error())
203+
}
204+
}
205+
return &reader{
206+
inputName: inputName,
207+
r: bufio.NewReader(r),
208+
decl: decl,
209+
xpath: expr,
210+
root: idr.CreateNode(idr.DocumentNode, "#root"),
211+
line: 1,
212+
}, nil
213+
}

‎extensions/omniv21/fileformat/fixedlength/reader_test.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package fixedlength
22

33
import (
44
"bufio"
5+
"encoding/json"
56
"errors"
67
"io"
78
"strings"
@@ -455,3 +456,38 @@ func TestRead_ByHeaderFooter(t *testing.T) {
455456
assert.Equal(t, io.EOF, err)
456457
assert.Nil(t, n)
457458
}
459+
460+
func TestRelease(t *testing.T) {
461+
var decl fileDecl
462+
err := json.Unmarshal([]byte(`
463+
{
464+
"envelopes": [
465+
{ "name": "env1", "columns": [ { "name": "col1", "start_pos": 1, "length": 3 } ] }
466+
]
467+
}`), &decl)
468+
assert.NoError(t, err)
469+
r, err := NewReader("test", strings.NewReader("abcd\n12\n#$%^\n"), &decl, ".[starts-with(., '1')]")
470+
assert.NoError(t, err)
471+
n, err := r.Read()
472+
assert.NoError(t, err)
473+
assert.Equal(t, `{"col1":"12"}`, idr.JSONify2(n))
474+
assert.Equal(t, `{"env1":{"col1":"12"}}`, idr.JSONify2(r.root))
475+
assert.True(t, n == r.target)
476+
r.Release(n)
477+
assert.Nil(t, r.target)
478+
assert.Equal(t, `{}`, idr.JSONify2(r.root))
479+
}
480+
481+
func TestIsContinuableError(t *testing.T) {
482+
r := &reader{}
483+
assert.True(t, r.IsContinuableError(r.FmtErr("some error")))
484+
assert.False(t, r.IsContinuableError(ErrInvalidEnvelope("invalid envelope")))
485+
assert.False(t, r.IsContinuableError(io.EOF))
486+
}
487+
488+
func TestNewReader(t *testing.T) {
489+
r, err := NewReader("test", nil, nil, "[invalid")
490+
assert.Error(t, err)
491+
assert.Equal(t, `invalid xpath '[invalid', err: expression must evaluate to a node-set`, err.Error())
492+
assert.Nil(t, r)
493+
}

‎extensions/omniv21/schemahandler.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
99
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/csv"
1010
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/edi"
11+
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/fixedlength"
1112
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/json"
1213
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/xml"
1314
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
@@ -84,10 +85,10 @@ func customParseFuncs(ctx *schemahandler.CreateCtx) transform.CustomParseFuncs {
8485
func fileFormats(ctx *schemahandler.CreateCtx) []fileformat.FileFormat {
8586
formats := []fileformat.FileFormat{
8687
csv.NewCSVFileFormat(ctx.Name),
88+
edi.NewEDIFileFormat(ctx.Name),
89+
fixedlength.NewFixedLengthFileFormat(ctx.Name),
8790
json.NewJSONFileFormat(ctx.Name),
8891
xml.NewXMLFileFormat(ctx.Name),
89-
edi.NewEDIFileFormat(ctx.Name),
90-
// TODO more built-in file formats to come.
9192
}
9293
if ctx.CreateParams == nil {
9394
return formats

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /